Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Newer
Older
100644 773 lines (686 sloc) 27.823 kb
45d65cf Renamed s3py to s3cmd
mludvig authored
1 ## Amazon S3 manager
2 ## Author: Michal Ludvig <michal@logix.cz>
3 ## http://www.logix.cz/michal
4 ## License: GPL Version 2
5
a580bd1 - New feature - allow "get" to stdout
mludvig authored
6 import sys
45d65cf Renamed s3py to s3cmd
mludvig authored
7 import os, os.path
a3f63be * s3/s3.py: improved retrying in send_request() and send_file()
ludvigm authored
8 import time
45d65cf Renamed s3py to s3cmd
mludvig authored
9 import httplib
10 import logging
1202afb 2008-02-11 Michal Ludvig <michal@logix.cz>
ludvigm authored
11 import mimetypes
5c3554d * s3cmd, S3/AccessLog.py, ...: Added [accesslog] command.
ludvigm authored
12 import re
45d65cf Renamed s3py to s3cmd
mludvig authored
13 from logging import debug, info, warning, error
14 from stat import ST_SIZE
15
4ffe940 * S3/S3.py, S3/Utils.py: Use 'hashlib' instead of md5 and sha
w_tell authored
16 try:
1ccadf2 * s3cmd: New command [sign]
ludvigm authored
17 from hashlib import md5
4ffe940 * S3/S3.py, S3/Utils.py: Use 'hashlib' instead of md5 and sha
w_tell authored
18 except ImportError:
ac754aa * S3/S3.py, S3/Utils.py: Use 'hashlib' instead of md5 and sha
w_tell authored
19 from md5 import md5
4ffe940 * S3/S3.py, S3/Utils.py: Use 'hashlib' instead of md5 and sha
w_tell authored
20
45d65cf Renamed s3py to s3cmd
mludvig authored
21 from Utils import *
22 from SortedDict import SortedDict
23 from BidirMap import BidirMap
24 from Config import Config
5415fba 2008-04-29 Michal Ludvig <michal@logix.cz>
ludvigm authored
25 from Exceptions import *
5c3554d * s3cmd, S3/AccessLog.py, ...: Added [accesslog] command.
ludvigm authored
26 from ACL import ACL, GranteeLogDelivery
27 from AccessLog import AccessLog
28 from S3Uri import S3Uri
389b582 Versioning support, and a fix to remote2remote that corrects remote path...
Sundar Raman authored
29 from Versioning import Versioning
45d65cf Renamed s3py to s3cmd
mludvig authored
30
5c3554d * s3cmd, S3/AccessLog.py, ...: Added [accesslog] command.
ludvigm authored
31 __all__ = []
24f6313 * S3/S3.py: Re-sign requests before retrial to avoid
ludvigm authored
32 class S3Request(object):
33 def __init__(self, s3, method_string, resource, headers, params = {}):
34 self.s3 = s3
b33b25f 2009-05-27 Michal Ludvig <michal@logix.cz>
ludvigm authored
35 self.headers = SortedDict(headers or {}, ignore_case = True)
24f6313 * S3/S3.py: Re-sign requests before retrial to avoid
ludvigm authored
36 self.resource = resource
37 self.method_string = method_string
38 self.params = params
39
40 self.update_timestamp()
41 self.sign()
42
43 def update_timestamp(self):
44 if self.headers.has_key("date"):
45 del(self.headers["date"])
46 self.headers["x-amz-date"] = time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime())
47
48 def format_param_str(self):
49 """
50 Format URL parameters from self.params and returns
51 ?parm1=val1&parm2=val2 or an empty string if there
52 are no parameters. Output of this function should
53 be appended directly to self.resource['uri']
54 """
55 param_str = ""
56 for param in self.params:
57 if self.params[param] not in (None, ""):
58 param_str += "&%s=%s" % (param, self.params[param])
59 else:
60 param_str += "&%s" % param
61 return param_str and "?" + param_str[1:]
62
63 def sign(self):
64 h = self.method_string + "\n"
65 h += self.headers.get("content-md5", "")+"\n"
66 h += self.headers.get("content-type", "")+"\n"
67 h += self.headers.get("date", "")+"\n"
68 for header in self.headers.keys():
69 if header.startswith("x-amz-"):
70 h += header+":"+str(self.headers[header])+"\n"
71 if self.resource['bucket']:
72 h += "/" + self.resource['bucket']
73 h += self.resource['uri']
74 debug("SignHeaders: " + repr(h))
75 signature = sign_string(h)
76
77 self.headers["Authorization"] = "AWS "+self.s3.config.access_key+":"+signature
78
79 def get_triplet(self):
80 self.update_timestamp()
81 self.sign()
82 resource = dict(self.resource) ## take a copy
83 resource['uri'] += self.format_param_str()
84 return (self.method_string, resource, self.headers)
85
45d65cf Renamed s3py to s3cmd
mludvig authored
86 class S3(object):
87 http_methods = BidirMap(
88 GET = 0x01,
89 PUT = 0x02,
90 HEAD = 0x04,
91 DELETE = 0x08,
92 MASK = 0x0F,
93 )
94
95 targets = BidirMap(
96 SERVICE = 0x0100,
97 BUCKET = 0x0200,
98 OBJECT = 0x0400,
99 MASK = 0x0700,
100 )
101
102 operations = BidirMap(
103 UNDFINED = 0x0000,
104 LIST_ALL_BUCKETS = targets["SERVICE"] | http_methods["GET"],
105 BUCKET_CREATE = targets["BUCKET"] | http_methods["PUT"],
106 BUCKET_LIST = targets["BUCKET"] | http_methods["GET"],
107 BUCKET_DELETE = targets["BUCKET"] | http_methods["DELETE"],
108 OBJECT_PUT = targets["OBJECT"] | http_methods["PUT"],
109 OBJECT_GET = targets["OBJECT"] | http_methods["GET"],
110 OBJECT_HEAD = targets["OBJECT"] | http_methods["HEAD"],
111 OBJECT_DELETE = targets["OBJECT"] | http_methods["DELETE"],
112 )
113
114 codes = {
115 "NoSuchBucket" : "Bucket '%s' does not exist",
116 "AccessDenied" : "Access to bucket '%s' was denied",
117 "BucketAlreadyExists" : "Bucket '%s' already exists",
118 }
119
3ecbb36 2008-02-11 Michal Ludvig <michal@logix.cz>
ludvigm authored
120 ## S3 sometimes sends HTTP-307 response
121 redir_map = {}
122
a3f63be * s3/s3.py: improved retrying in send_request() and send_file()
ludvigm authored
123 ## Maximum attempts of re-issuing failed requests
124 _max_retries = 5
125
45d65cf Renamed s3py to s3cmd
mludvig authored
126 def __init__(self, config):
127 self.config = config
128
24f188c * S3/S3.py: Support for buckets stored in Europe, access now
ludvigm authored
129 def get_connection(self, bucket):
1fb8944 2007-07-05 Michal Ludvig <michal@logix.cz>
ludvigm authored
130 if self.config.proxy_host != "":
131 return httplib.HTTPConnection(self.config.proxy_host, self.config.proxy_port)
132 else:
24f188c * S3/S3.py: Support for buckets stored in Europe, access now
ludvigm authored
133 if self.config.use_https:
134 return httplib.HTTPSConnection(self.get_hostname(bucket))
135 else:
136 return httplib.HTTPConnection(self.get_hostname(bucket))
1fb8944 2007-07-05 Michal Ludvig <michal@logix.cz>
ludvigm authored
137
24f188c * S3/S3.py: Support for buckets stored in Europe, access now
ludvigm authored
138 def get_hostname(self, bucket):
d145044 * s3cmd, S3/CloudFront.py, S3/Config.py: Support access
ludvigm authored
139 if bucket and check_bucket_name_dns_conformity(bucket):
3ecbb36 2008-02-11 Michal Ludvig <michal@logix.cz>
ludvigm authored
140 if self.redir_map.has_key(bucket):
141 host = self.redir_map[bucket]
142 else:
d145044 * s3cmd, S3/CloudFront.py, S3/Config.py: Support access
ludvigm authored
143 host = getHostnameFromBucket(bucket)
24f188c * S3/S3.py: Support for buckets stored in Europe, access now
ludvigm authored
144 else:
145 host = self.config.host_base
3ecbb36 2008-02-11 Michal Ludvig <michal@logix.cz>
ludvigm authored
146 debug('get_hostname(%s): %s' % (bucket, host))
24f188c * S3/S3.py: Support for buckets stored in Europe, access now
ludvigm authored
147 return host
1fb8944 2007-07-05 Michal Ludvig <michal@logix.cz>
ludvigm authored
148
3ecbb36 2008-02-11 Michal Ludvig <michal@logix.cz>
ludvigm authored
149 def set_hostname(self, bucket, redir_hostname):
150 self.redir_map[bucket] = redir_hostname
151
24f188c * S3/S3.py: Support for buckets stored in Europe, access now
ludvigm authored
152 def format_uri(self, resource):
d145044 * s3cmd, S3/CloudFront.py, S3/Config.py: Support access
ludvigm authored
153 if resource['bucket'] and not check_bucket_name_dns_conformity(resource['bucket']):
c301244 * s3cmd, S3/S3.py, S3/Config.py: Allow access to upper-case
ludvigm authored
154 uri = "/%s%s" % (resource['bucket'], resource['uri'])
24f188c * S3/S3.py: Support for buckets stored in Europe, access now
ludvigm authored
155 else:
156 uri = resource['uri']
c301244 * s3cmd, S3/S3.py, S3/Config.py: Allow access to upper-case
ludvigm authored
157 if self.config.proxy_host != "":
158 uri = "http://%s%s" % (self.get_hostname(resource['bucket']), uri)
24f188c * S3/S3.py: Support for buckets stored in Europe, access now
ludvigm authored
159 debug('format_uri(): ' + uri)
160 return uri
1fb8944 2007-07-05 Michal Ludvig <michal@logix.cz>
ludvigm authored
161
45d65cf Renamed s3py to s3cmd
mludvig authored
162 ## Commands / Actions
163 def list_all_buckets(self):
164 request = self.create_request("LIST_ALL_BUCKETS")
165 response = self.send_request(request)
166 response["list"] = getListFromXml(response["data"], "Bucket")
167 return response
168
2de083a Fixes for non-recursive 'ls' support from previous commit.
ludvigm authored
169 def bucket_list(self, bucket, prefix = None, recursive = None):
169af58 * S3/S3.py: Support for buckets with over 1000 objects.
ludvigm authored
170 def _list_truncated(data):
925f2de 2007-09-25 Michal Ludvig <michal@logix.cz>
ludvigm authored
171 ## <IsTruncated> can either be "true" or "false" or be missing completely
172 is_truncated = getTextFromXml(data, ".//IsTruncated") or "false"
173 return is_truncated.lower() != "false"
169af58 * S3/S3.py: Support for buckets with over 1000 objects.
ludvigm authored
174
175 def _get_contents(data):
176 return getListFromXml(data, "Contents")
177
9cb9400 * s3cmd, S3/S3.py, NEWS: Support for (non-)recursive 'ls'
ludvigm authored
178 def _get_common_prefixes(data):
179 return getListFromXml(data, "CommonPrefixes")
180
181 uri_params = {}
5028177 * S3/S3.py: Fix bucket listing for buckets with
ludvigm authored
182 truncated = True
183 list = []
184 prefixes = []
185
186 while truncated:
8ecdd66 * s3cmd: New [fixbucket] command for fixing invalid object
ludvigm authored
187 response = self.bucket_list_noparse(bucket, prefix, recursive, uri_params)
240db6c * S3/S3.py: Fixed code formating.
ludvigm authored
188 current_list = _get_contents(response["data"])
189 current_prefixes = _get_common_prefixes(response["data"])
5028177 * S3/S3.py: Fix bucket listing for buckets with
ludvigm authored
190 truncated = _list_truncated(response["data"])
191 if truncated:
240db6c * S3/S3.py: Fixed code formating.
ludvigm authored
192 if current_list:
193 uri_params['marker'] = self.urlencode_string(current_list[-1]["Key"])
5028177 * S3/S3.py: Fix bucket listing for buckets with
ludvigm authored
194 else:
240db6c * S3/S3.py: Fixed code formating.
ludvigm authored
195 uri_params['marker'] = self.urlencode_string(current_prefixes[-1]["Prefix"])
5028177 * S3/S3.py: Fix bucket listing for buckets with
ludvigm authored
196 debug("Listing continues after '%s'" % uri_params['marker'])
197
240db6c * S3/S3.py: Fixed code formating.
ludvigm authored
198 list += current_list
199 prefixes += current_prefixes
5028177 * S3/S3.py: Fix bucket listing for buckets with
ludvigm authored
200
169af58 * S3/S3.py: Support for buckets with over 1000 objects.
ludvigm authored
201 response['list'] = list
9cb9400 * s3cmd, S3/S3.py, NEWS: Support for (non-)recursive 'ls'
ludvigm authored
202 response['common_prefixes'] = prefixes
45d65cf Renamed s3py to s3cmd
mludvig authored
203 return response
204
8ecdd66 * s3cmd: New [fixbucket] command for fixing invalid object
ludvigm authored
205 def bucket_list_noparse(self, bucket, prefix = None, recursive = None, uri_params = {}):
206 if prefix:
207 uri_params['prefix'] = self.urlencode_string(prefix)
208 if not self.config.recursive and not recursive:
209 uri_params['delimiter'] = "/"
210 request = self.create_request("BUCKET_LIST", bucket = bucket, **uri_params)
211 response = self.send_request(request)
212 #debug(response)
213 return response
214
24f188c * S3/S3.py: Support for buckets stored in Europe, access now
ludvigm authored
215 def bucket_create(self, bucket, bucket_location = None):
b33b25f 2009-05-27 Michal Ludvig <michal@logix.cz>
ludvigm authored
216 headers = SortedDict(ignore_case = True)
24f188c * S3/S3.py: Support for buckets stored in Europe, access now
ludvigm authored
217 body = ""
218 if bucket_location and bucket_location.strip().upper() != "US":
627aa58 * s3cmd, S3/S3.py: Added support for bucket locations
ludvigm authored
219 bucket_location = bucket_location.strip()
220 if bucket_location.upper() == "EU":
221 bucket_location = bucket_location.upper()
222 else:
223 bucket_location = bucket_location.lower()
24f188c * S3/S3.py: Support for buckets stored in Europe, access now
ludvigm authored
224 body = "<CreateBucketConfiguration><LocationConstraint>"
627aa58 * s3cmd, S3/S3.py: Added support for bucket locations
ludvigm authored
225 body += bucket_location
24f188c * S3/S3.py: Support for buckets stored in Europe, access now
ludvigm authored
226 body += "</LocationConstraint></CreateBucketConfiguration>"
227 debug("bucket_location: " + body)
d145044 * s3cmd, S3/CloudFront.py, S3/Config.py: Support access
ludvigm authored
228 check_bucket_name(bucket, dns_strict = True)
306a4a7 * S3/S3.py: "s3cmd mb" can create upper-case buckets again
ludvigm authored
229 else:
d145044 * s3cmd, S3/CloudFront.py, S3/Config.py: Support access
ludvigm authored
230 check_bucket_name(bucket, dns_strict = False)
a878770 2008-02-11 Michal Ludvig <michal@logix.cz>
ludvigm authored
231 if self.config.acl_public:
232 headers["x-amz-acl"] = "public-read"
1fb8944 2007-07-05 Michal Ludvig <michal@logix.cz>
ludvigm authored
233 request = self.create_request("BUCKET_CREATE", bucket = bucket, headers = headers)
24f188c * S3/S3.py: Support for buckets stored in Europe, access now
ludvigm authored
234 response = self.send_request(request, body)
45d65cf Renamed s3py to s3cmd
mludvig authored
235 return response
236
237 def bucket_delete(self, bucket):
238 request = self.create_request("BUCKET_DELETE", bucket = bucket)
239 response = self.send_request(request)
240 return response
241
d8a573e 2008-02-27 Michal Ludvig <michal@logix.cz>
ludvigm authored
242 def bucket_info(self, uri):
243 request = self.create_request("BUCKET_LIST", bucket = uri.bucket(), extra = "?location")
889b7c1 * s3cmd, S3/S3.py: New command 'ib' to get information about
ludvigm authored
244 response = self.send_request(request)
24f188c * S3/S3.py: Support for buckets stored in Europe, access now
ludvigm authored
245 response['bucket-location'] = getTextFromXml(response['data'], "LocationConstraint") or "any"
889b7c1 * s3cmd, S3/S3.py: New command 'ib' to get information about
ludvigm authored
246 return response
247
154795f * s3cmd, S3/S3.py, S3/Progress.py: Display "[X of Y]"
ludvigm authored
248 def object_put(self, filename, uri, extra_headers = None, extra_label = ""):
26f22ff * S3/S3.py: removed object_{get,put,delete}_uri() functions
ludvigm authored
249 # TODO TODO
250 # Make it consistent with stream-oriented object_get()
251 if uri.type != "s3":
252 raise ValueError("Expected URI type 's3', got '%s'" % uri.type)
253
45d65cf Renamed s3py to s3cmd
mludvig authored
254 if not os.path.isfile(filename):
1add49f * s3cmd, S3/Utils.py, S3/Exceptions.py, S3/Progress.py,
ludvigm authored
255 raise InvalidFileError(u"%s is not a regular file" % unicodise(filename))
45d65cf Renamed s3py to s3cmd
mludvig authored
256 try:
cf51d50 * S3/S3.py, S3/Utils.py: open files in binary mode (otherwise windows
ludvigm authored
257 file = open(filename, "rb")
45d65cf Renamed s3py to s3cmd
mludvig authored
258 size = os.stat(filename)[ST_SIZE]
2edf0e8 * s3cmd, S3/S3.py: Ignore inaccessible (and missing) files
ludvigm authored
259 except (IOError, OSError), e:
1add49f * s3cmd, S3/Utils.py, S3/Exceptions.py, S3/Progress.py,
ludvigm authored
260 raise InvalidFileError(u"%s: %s" % (unicodise(filename), e.strerror))
b33b25f 2009-05-27 Michal Ludvig <michal@logix.cz>
ludvigm authored
261 headers = SortedDict(ignore_case = True)
fd56bd5 2007-05-27 Michal Ludvig <michal@logix.cz>
ludvigm authored
262 if extra_headers:
263 headers.update(extra_headers)
45d65cf Renamed s3py to s3cmd
mludvig authored
264 headers["content-length"] = size
1202afb 2008-02-11 Michal Ludvig <michal@logix.cz>
ludvigm authored
265 content_type = None
266 if self.config.guess_mime_type:
267 content_type = mimetypes.guess_type(filename)[0]
268 if not content_type:
269 content_type = self.config.default_mime_type
270 debug("Content-Type set to '%s'" % content_type)
271 headers["content-type"] = content_type
45d65cf Renamed s3py to s3cmd
mludvig authored
272 if self.config.acl_public:
273 headers["x-amz-acl"] = "public-read"
db08664 * s3cmd, S3/S3.py, S3/Config.py: Added --reduced-redundancy
ludvigm authored
274 if self.config.reduced_redundancy:
275 headers["x-amz-storage-class"] = "REDUCED_REDUNDANCY"
26f22ff * S3/S3.py: removed object_{get,put,delete}_uri() functions
ludvigm authored
276 request = self.create_request("OBJECT_PUT", uri = uri, headers = headers)
1add49f * s3cmd, S3/Utils.py, S3/Exceptions.py, S3/Progress.py,
ludvigm authored
277 labels = { 'source' : unicodise(filename), 'destination' : unicodise(uri.uri()), 'extra' : extra_label }
4617f44 * s3cmd: Don't display download/upload completed message
ludvigm authored
278 response = self.send_file(request, file, labels)
45d65cf Renamed s3py to s3cmd
mludvig authored
279 return response
280
154795f * s3cmd, S3/S3.py, S3/Progress.py: Display "[X of Y]"
ludvigm authored
281 def object_get(self, uri, stream, start_position = 0, extra_label = ""):
0f4094f Implemented S3->local sync
ludvigm authored
282 if uri.type != "s3":
283 raise ValueError("Expected URI type 's3', got '%s'" % uri.type)
26f22ff * S3/S3.py: removed object_{get,put,delete}_uri() functions
ludvigm authored
284 request = self.create_request("OBJECT_GET", uri = uri)
557af82 * run-tests.py: Updated paths for the new sync
ludvigm authored
285 labels = { 'source' : unicodise(uri.uri()), 'destination' : unicodise(stream.name), 'extra' : extra_label }
4617f44 * s3cmd: Don't display download/upload completed message
ludvigm authored
286 response = self.recv_file(request, stream, labels, start_position)
45d65cf Renamed s3py to s3cmd
mludvig authored
287 return response
0f4094f Implemented S3->local sync
ludvigm authored
288
26f22ff * S3/S3.py: removed object_{get,put,delete}_uri() functions
ludvigm authored
289 def object_delete(self, uri):
45d65cf Renamed s3py to s3cmd
mludvig authored
290 if uri.type != "s3":
291 raise ValueError("Expected URI type 's3', got '%s'" % uri.type)
26f22ff * S3/S3.py: removed object_{get,put,delete}_uri() functions
ludvigm authored
292 request = self.create_request("OBJECT_DELETE", uri = uri)
293 response = self.send_request(request)
294 return response
45d65cf Renamed s3py to s3cmd
mludvig authored
295
e538388 * s3cmd: Support for 'cp' command.
ludvigm authored
296 def object_copy(self, src_uri, dst_uri, extra_headers = None):
297 if src_uri.type != "s3":
298 raise ValueError("Expected URI type 's3', got '%s'" % src_uri.type)
299 if dst_uri.type != "s3":
300 raise ValueError("Expected URI type 's3', got '%s'" % dst_uri.type)
b33b25f 2009-05-27 Michal Ludvig <michal@logix.cz>
ludvigm authored
301 headers = SortedDict(ignore_case = True)
e538388 * s3cmd: Support for 'cp' command.
ludvigm authored
302 headers['x-amz-copy-source'] = "/%s/%s" % (src_uri.bucket(), self.urlencode_string(src_uri.object()))
aa63aa3 * s3cmd: Support for recursive [cp] and [mv], including
ludvigm authored
303 ## TODO: For now COPY, later maybe add a switch?
304 headers['x-amz-metadata-directive'] = "COPY"
e538388 * s3cmd: Support for 'cp' command.
ludvigm authored
305 if self.config.acl_public:
306 headers["x-amz-acl"] = "public-read"
db08664 * s3cmd, S3/S3.py, S3/Config.py: Added --reduced-redundancy
ludvigm authored
307 if self.config.reduced_redundancy:
308 headers["x-amz-storage-class"] = "REDUCED_REDUNDANCY"
aa63aa3 * s3cmd: Support for recursive [cp] and [mv], including
ludvigm authored
309 # if extra_headers:
310 # headers.update(extra_headers)
e538388 * s3cmd: Support for 'cp' command.
ludvigm authored
311 request = self.create_request("OBJECT_PUT", uri = dst_uri, headers = headers)
312 response = self.send_request(request)
313 return response
314
1e24254 * s3cmd, S3/S3.py, NEWS: "s3cmd mv" for moving objects
ludvigm authored
315 def object_move(self, src_uri, dst_uri, extra_headers = None):
316 response_copy = self.object_copy(src_uri, dst_uri, extra_headers)
317 debug("Object %s copied to %s" % (src_uri, dst_uri))
318 if getRootTagName(response_copy["data"]) == "CopyObjectResult":
319 response_delete = self.object_delete(src_uri)
320 debug("Object %s deleted" % src_uri)
321 return response_copy
322
d8a573e 2008-02-27 Michal Ludvig <michal@logix.cz>
ludvigm authored
323 def object_info(self, uri):
26f22ff * S3/S3.py: removed object_{get,put,delete}_uri() functions
ludvigm authored
324 request = self.create_request("OBJECT_HEAD", uri = uri)
d8a573e 2008-02-27 Michal Ludvig <michal@logix.cz>
ludvigm authored
325 response = self.send_request(request)
326 return response
327
328 def get_acl(self, uri):
329 if uri.has_object():
26f22ff * S3/S3.py: removed object_{get,put,delete}_uri() functions
ludvigm authored
330 request = self.create_request("OBJECT_GET", uri = uri, extra = "?acl")
d8a573e 2008-02-27 Michal Ludvig <michal@logix.cz>
ludvigm authored
331 else:
332 request = self.create_request("BUCKET_LIST", bucket = uri.bucket(), extra = "?acl")
6ceb3e1 * S3/ACL.py: New object for handling ACL issues.
ludvigm authored
333
d8a573e 2008-02-27 Michal Ludvig <michal@logix.cz>
ludvigm authored
334 response = self.send_request(request)
6ceb3e1 * S3/ACL.py: New object for handling ACL issues.
ludvigm authored
335 acl = ACL(response['data'])
b3b13bd * s3cmd: Factored remote_keys generation from cmd_object_get()
ludvigm authored
336 return acl
d8a573e 2008-02-27 Michal Ludvig <michal@logix.cz>
ludvigm authored
337
f461af3 * s3cmd: New command 'setacl'.
ludvigm authored
338 def set_acl(self, uri, acl):
339 if uri.has_object():
340 request = self.create_request("OBJECT_PUT", uri = uri, extra = "?acl")
341 else:
342 request = self.create_request("BUCKET_CREATE", bucket = uri.bucket(), extra = "?acl")
343
344 body = str(acl)
345 debug(u"set_acl(%s): acl-xml: %s" % (uri, body))
346 response = self.send_request(request, body)
347 return response
348
5c3554d * s3cmd, S3/AccessLog.py, ...: Added [accesslog] command.
ludvigm authored
349 def get_accesslog(self, uri):
350 request = self.create_request("BUCKET_LIST", bucket = uri.bucket(), extra = "?logging")
351 response = self.send_request(request)
352 accesslog = AccessLog(response['data'])
353 return accesslog
354
355 def set_accesslog_acl(self, uri):
356 acl = self.get_acl(uri)
357 debug("Current ACL(%s): %s" % (uri.uri(), str(acl)))
358 acl.appendGrantee(GranteeLogDelivery("READ_ACP"))
359 acl.appendGrantee(GranteeLogDelivery("WRITE"))
360 debug("Updated ACL(%s): %s" % (uri.uri(), str(acl)))
361 self.set_acl(uri, acl)
362
363 def set_accesslog(self, uri, enable, log_target_prefix_uri = None, acl_public = False):
364 request = self.create_request("BUCKET_CREATE", bucket = uri.bucket(), extra = "?logging")
365 accesslog = AccessLog()
366 if enable:
367 accesslog.enableLogging(log_target_prefix_uri)
368 accesslog.setAclPublic(acl_public)
369 else:
370 accesslog.disableLogging()
371 body = str(accesslog)
372 debug(u"set_accesslog(%s): accesslog-xml: %s" % (uri, body))
373 try:
374 response = self.send_request(request, body)
375 except S3Error, e:
376 if e.info['Code'] == "InvalidTargetBucketForLogging":
377 info("Setting up log-delivery ACL for target bucket.")
378 self.set_accesslog_acl(S3Uri("s3://%s" % log_target_prefix_uri.bucket()))
379 response = self.send_request(request, body)
380 else:
381 raise
382 return accesslog, response
383
389b582 Versioning support, and a fix to remote2remote that corrects remote path...
Sundar Raman authored
384 # Implement versioning support - get and set
385 def get_versioning(self, uri):
386 request = self.create_request("BUCKET_LIST", bucket = uri.bucket(), extra = "?versioning")
387 response = self.send_request(request)
388 print "Response: %s" % response
389 versioning = Versioning(response['data'])
390 return versioning
391
392 def set_versioning(self, uri):
393 pass
394
395
45d65cf Renamed s3py to s3cmd
mludvig authored
396 ## Low level methods
8ecdd66 * s3cmd: New [fixbucket] command for fixing invalid object
ludvigm authored
397 def urlencode_string(self, string, urlencoding_mode = None):
8b940d1 * s3cmd, S3/S3Uri.py, S3/S3.py: All internal representations of
ludvigm authored
398 if type(string) == unicode:
399 string = string.encode("utf-8")
a68006e * s3cmd, S3/Config.py, S3/S3.py: Support for --verbatim.
ludvigm authored
400
8ecdd66 * s3cmd: New [fixbucket] command for fixing invalid object
ludvigm authored
401 if urlencoding_mode is None:
402 urlencoding_mode = self.config.urlencoding_mode
403
404 if urlencoding_mode == "verbatim":
a68006e * s3cmd, S3/Config.py, S3/S3.py: Support for --verbatim.
ludvigm authored
405 ## Don't do any pre-processing
406 return string
407
8192aed 2007-08-13 Michal Ludvig <michal@logix.cz>
ludvigm authored
408 encoded = ""
409 ## List of characters that must be escaped for S3
410 ## Haven't found this in any official docs
411 ## but my tests show it's more less correct.
412 ## If you start getting InvalidSignature errors
413 ## from S3 check the error headers returned
414 ## from S3 to see whether the list hasn't
415 ## changed.
416 for c in string: # I'm not sure how to know in what encoding
417 # 'object' is. Apparently "type(object)==str"
418 # but the contents is a string of unicode
419 # bytes, e.g. '\xc4\x8d\xc5\xafr\xc3\xa1k'
420 # Don't know what it will do on non-utf8
421 # systems.
422 # [hope that sounds reassuring ;-)]
423 o = ord(c)
dbb4048 * S3/Utils.py: New function replace_nonprintables()
ludvigm authored
424 if (o < 0x20 or o == 0x7f):
8ecdd66 * s3cmd: New [fixbucket] command for fixing invalid object
ludvigm authored
425 if urlencoding_mode == "fixbucket":
426 encoded += "%%%02X" % o
427 else:
428 error(u"Non-printable character 0x%02x in: %s" % (o, string))
429 error(u"Please report it to s3tools-bugs@lists.sourceforge.net")
430 encoded += replace_nonprintables(c)
dbb4048 * S3/Utils.py: New function replace_nonprintables()
ludvigm authored
431 elif (o == 0x20 or # Space and below
8192aed 2007-08-13 Michal Ludvig <michal@logix.cz>
ludvigm authored
432 o == 0x22 or # "
433 o == 0x23 or # #
dbb4048 * S3/Utils.py: New function replace_nonprintables()
ludvigm authored
434 o == 0x25 or # % (escape character)
a68006e * s3cmd, S3/Config.py, S3/S3.py: Support for --verbatim.
ludvigm authored
435 o == 0x26 or # &
8192aed 2007-08-13 Michal Ludvig <michal@logix.cz>
ludvigm authored
436 o == 0x2B or # + (or it would become <space>)
437 o == 0x3C or # <
438 o == 0x3E or # >
439 o == 0x3F or # ?
440 o == 0x60 or # `
441 o >= 123): # { and above, including >= 128 for UTF-8
442 encoded += "%%%02X" % o
443 else:
444 encoded += c
445 debug("String '%s' encoded to '%s'" % (string, encoded))
446 return encoded
447
26f22ff * S3/S3.py: removed object_{get,put,delete}_uri() functions
ludvigm authored
448 def create_request(self, operation, uri = None, bucket = None, object = None, headers = None, extra = None, **params):
24f188c * S3/S3.py: Support for buckets stored in Europe, access now
ludvigm authored
449 resource = { 'bucket' : None, 'uri' : "/" }
26f22ff * S3/S3.py: removed object_{get,put,delete}_uri() functions
ludvigm authored
450
451 if uri and (bucket or object):
452 raise ValueError("Both 'uri' and either 'bucket' or 'object' parameters supplied")
453 ## If URI is given use that instead of bucket/object parameters
454 if uri:
455 bucket = uri.bucket()
456 object = uri.has_object() and uri.object() or None
457
45d65cf Renamed s3py to s3cmd
mludvig authored
458 if bucket:
24f188c * S3/S3.py: Support for buckets stored in Europe, access now
ludvigm authored
459 resource['bucket'] = str(bucket)
45d65cf Renamed s3py to s3cmd
mludvig authored
460 if object:
24f188c * S3/S3.py: Support for buckets stored in Europe, access now
ludvigm authored
461 resource['uri'] = "/" + self.urlencode_string(object)
462 if extra:
463 resource['uri'] += extra
45d65cf Renamed s3py to s3cmd
mludvig authored
464
24f6313 * S3/S3.py: Re-sign requests before retrial to avoid
ludvigm authored
465 method_string = S3.http_methods.getkey(S3.operations[operation] & S3.http_methods["MASK"])
f95a310 * s3cmd, S3/Config.py, S3/S3.py: Added --add-header option.
ludvigm authored
466
24f6313 * S3/S3.py: Re-sign requests before retrial to avoid
ludvigm authored
467 request = S3Request(self, method_string, resource, headers, params)
45d65cf Renamed s3py to s3cmd
mludvig authored
468
24f188c * S3/S3.py: Support for buckets stored in Europe, access now
ludvigm authored
469 debug("CreateRequest: resource[uri]=" + resource['uri'])
24f6313 * S3/S3.py: Re-sign requests before retrial to avoid
ludvigm authored
470 return request
45d65cf Renamed s3py to s3cmd
mludvig authored
471
a3f63be * s3/s3.py: improved retrying in send_request() and send_file()
ludvigm authored
472 def _fail_wait(self, retries):
473 # Wait a few seconds. The more it fails the more we wait.
474 return (self._max_retries - retries + 1) * 3
475
476 def send_request(self, request, body = None, retries = _max_retries):
24f6313 * S3/S3.py: Re-sign requests before retrial to avoid
ludvigm authored
477 method_string, resource, headers = request.get_triplet()
20cb0e0 * s3cmd, S3/S3.py: Make --verbose mode more useful and default
ludvigm authored
478 debug("Processing request, please wait...")
789633b * s3cmd: And send it for requests with no body as well...
ludvigm authored
479 if not headers.has_key('content-length'):
480 headers['content-length'] = body and len(body) or 0
d3e0d2c Merge from 0.9.8.x branch, rel 249:
ludvigm authored
481 try:
f8c06b8 * S3/S3.py: "Stringify" all headers. Httplib should do
ludvigm authored
482 # "Stringify" all headers
483 for header in headers.keys():
7c9fd69 Doh! Test before commit, all right?
ludvigm authored
484 headers[header] = str(headers[header])
d3e0d2c Merge from 0.9.8.x branch, rel 249:
ludvigm authored
485 conn = self.get_connection(resource['bucket'])
486 conn.request(method_string, self.format_uri(resource), body, headers)
487 response = {}
488 http_response = conn.getresponse()
489 response["status"] = http_response.status
490 response["reason"] = http_response.reason
491 response["headers"] = convertTupleListToDict(http_response.getheaders())
492 response["data"] = http_response.read()
493 debug("Response: " + str(response))
494 conn.close()
f4aaaa9 Merge from 0.9.8.x branch, rel 251:
ludvigm authored
495 except Exception, e:
d3e0d2c Merge from 0.9.8.x branch, rel 249:
ludvigm authored
496 if retries:
f4aaaa9 Merge from 0.9.8.x branch, rel 251:
ludvigm authored
497 warning("Retrying failed request: %s (%s)" % (resource['uri'], e))
a3f63be * s3/s3.py: improved retrying in send_request() and send_file()
ludvigm authored
498 warning("Waiting %d sec..." % self._fail_wait(retries))
499 time.sleep(self._fail_wait(retries))
d3e0d2c Merge from 0.9.8.x branch, rel 249:
ludvigm authored
500 return self.send_request(request, body, retries - 1)
501 else:
502 raise S3RequestError("Request failed for: %s" % resource['uri'])
3ecbb36 2008-02-11 Michal Ludvig <michal@logix.cz>
ludvigm authored
503
504 if response["status"] == 307:
505 ## RedirectPermanent
506 redir_bucket = getTextFromXml(response['data'], ".//Bucket")
507 redir_hostname = getTextFromXml(response['data'], ".//Endpoint")
508 self.set_hostname(redir_bucket, redir_hostname)
20cb0e0 * s3cmd, S3/S3.py: Make --verbose mode more useful and default
ludvigm authored
509 warning("Redirected to: %s" % (redir_hostname))
3ecbb36 2008-02-11 Michal Ludvig <michal@logix.cz>
ludvigm authored
510 return self.send_request(request, body)
511
f4aaaa9 Merge from 0.9.8.x branch, rel 251:
ludvigm authored
512 if response["status"] >= 500:
513 e = S3Error(response)
d3e0d2c Merge from 0.9.8.x branch, rel 249:
ludvigm authored
514 if retries:
f4aaaa9 Merge from 0.9.8.x branch, rel 251:
ludvigm authored
515 warning(u"Retrying failed request: %s" % resource['uri'])
516 warning(unicode(e))
a3f63be * s3/s3.py: improved retrying in send_request() and send_file()
ludvigm authored
517 warning("Waiting %d sec..." % self._fail_wait(retries))
518 time.sleep(self._fail_wait(retries))
d3e0d2c Merge from 0.9.8.x branch, rel 249:
ludvigm authored
519 return self.send_request(request, body, retries - 1)
520 else:
f4aaaa9 Merge from 0.9.8.x branch, rel 251:
ludvigm authored
521 raise e
522
523 if response["status"] < 200 or response["status"] > 299:
524 raise S3Error(response)
525
45d65cf Renamed s3py to s3cmd
mludvig authored
526 return response
527
4617f44 * s3cmd: Don't display download/upload completed message
ludvigm authored
528 def send_file(self, request, file, labels, throttle = 0, retries = _max_retries):
24f6313 * S3/S3.py: Re-sign requests before retrial to avoid
ludvigm authored
529 method_string, resource, headers = request.get_triplet()
e411362 * S3/Progress.py: Two progress meter implementations.
ludvigm authored
530 size_left = size_total = headers.get("content-length")
531 if self.config.progress_meter:
4617f44 * s3cmd: Don't display download/upload completed message
ludvigm authored
532 progress = self.config.progress_class(labels, size_total)
e411362 * S3/Progress.py: Two progress meter implementations.
ludvigm authored
533 else:
534 info("Sending file '%s', please wait..." % file.name)
535 timestamp_start = time.time()
a3f63be * s3/s3.py: improved retrying in send_request() and send_file()
ludvigm authored
536 try:
537 conn = self.get_connection(resource['bucket'])
538 conn.connect()
539 conn.putrequest(method_string, self.format_uri(resource))
540 for header in headers.keys():
541 conn.putheader(header, str(headers[header]))
542 conn.endheaders()
543 except Exception, e:
4f469e9 * S3/S3.py, s3cmd, S3/Config.py, s3cmd.1: Added --continue for
ludvigm authored
544 if self.config.progress_meter:
545 progress.done("failed")
a3f63be * s3/s3.py: improved retrying in send_request() and send_file()
ludvigm authored
546 if retries:
547 warning("Retrying failed request: %s (%s)" % (resource['uri'], e))
548 warning("Waiting %d sec..." % self._fail_wait(retries))
549 time.sleep(self._fail_wait(retries))
550 # Connection error -> same throttle value
4617f44 * s3cmd: Don't display download/upload completed message
ludvigm authored
551 return self.send_file(request, file, labels, throttle, retries - 1)
a3f63be * s3/s3.py: improved retrying in send_request() and send_file()
ludvigm authored
552 else:
4f469e9 * S3/S3.py, s3cmd, S3/Config.py, s3cmd.1: Added --continue for
ludvigm authored
553 raise S3UploadError("Upload failed for: %s" % resource['uri'])
3ecbb36 2008-02-11 Michal Ludvig <michal@logix.cz>
ludvigm authored
554 file.seek(0)
4ffe940 * S3/S3.py, S3/Utils.py: Use 'hashlib' instead of md5 and sha
w_tell authored
555 md5_hash = md5()
a3f63be * s3/s3.py: improved retrying in send_request() and send_file()
ludvigm authored
556 try:
557 while (size_left > 0):
b67753b * S3/S3.py: Some errors during file upload were incorrectly
ludvigm authored
558 #debug("SendFile: Reading up to %d bytes from '%s'" % (self.config.send_chunk, file.name))
a3f63be * s3/s3.py: improved retrying in send_request() and send_file()
ludvigm authored
559 data = file.read(self.config.send_chunk)
560 md5_hash.update(data)
4f469e9 * S3/S3.py, s3cmd, S3/Config.py, s3cmd.1: Added --continue for
ludvigm authored
561 conn.send(data)
e411362 * S3/Progress.py: Two progress meter implementations.
ludvigm authored
562 if self.config.progress_meter:
a3f63be * s3/s3.py: improved retrying in send_request() and send_file()
ludvigm authored
563 progress.update(delta_position = len(data))
564 size_left -= len(data)
565 if throttle:
566 time.sleep(throttle)
567 md5_computed = md5_hash.hexdigest()
568 response = {}
569 http_response = conn.getresponse()
570 response["status"] = http_response.status
571 response["reason"] = http_response.reason
572 response["headers"] = convertTupleListToDict(http_response.getheaders())
573 response["data"] = http_response.read()
574 response["size"] = size_total
575 conn.close()
b67753b * S3/S3.py: Some errors during file upload were incorrectly
ludvigm authored
576 debug(u"Response: %s" % response)
a3f63be * s3/s3.py: improved retrying in send_request() and send_file()
ludvigm authored
577 except Exception, e:
578 if self.config.progress_meter:
579 progress.done("failed")
580 if retries:
408cb18 * Fixed reference to _max_retries
ludvigm authored
581 if retries < self._max_retries:
293a02b * S3/S3.py: Introduce throttling on upload only after
ludvigm authored
582 throttle = throttle and throttle * 5 or 0.01
4f469e9 * S3/S3.py, s3cmd, S3/Config.py, s3cmd.1: Added --continue for
ludvigm authored
583 warning("Upload failed: %s (%s)" % (resource['uri'], e))
a3f63be * s3/s3.py: improved retrying in send_request() and send_file()
ludvigm authored
584 warning("Retrying on lower speed (throttle=%0.2f)" % throttle)
585 warning("Waiting %d sec..." % self._fail_wait(retries))
586 time.sleep(self._fail_wait(retries))
587 # Connection error -> same throttle value
4617f44 * s3cmd: Don't display download/upload completed message
ludvigm authored
588 return self.send_file(request, file, labels, throttle, retries - 1)
a3f63be * s3/s3.py: improved retrying in send_request() and send_file()
ludvigm authored
589 else:
590 debug("Giving up on '%s' %s" % (file.name, e))
4f469e9 * S3/S3.py, s3cmd, S3/Config.py, s3cmd.1: Added --continue for
ludvigm authored
591 raise S3UploadError("Upload failed for: %s" % resource['uri'])
3ecbb36 2008-02-11 Michal Ludvig <michal@logix.cz>
ludvigm authored
592
e411362 * S3/Progress.py: Two progress meter implementations.
ludvigm authored
593 timestamp_end = time.time()
594 response["elapsed"] = timestamp_end - timestamp_start
595 response["speed"] = response["elapsed"] and float(response["size"]) / response["elapsed"] or float(-1)
596
597 if self.config.progress_meter:
598 ## The above conn.close() takes some time -> update() progress meter
599 ## to correct the average speed. Otherwise people will complain that
600 ## 'progress' and response["speed"] are inconsistent ;-)
601 progress.update()
602 progress.done("done")
603
3ecbb36 2008-02-11 Michal Ludvig <michal@logix.cz>
ludvigm authored
604 if response["status"] == 307:
605 ## RedirectPermanent
606 redir_bucket = getTextFromXml(response['data'], ".//Bucket")
607 redir_hostname = getTextFromXml(response['data'], ".//Endpoint")
608 self.set_hostname(redir_bucket, redir_hostname)
20cb0e0 * s3cmd, S3/S3.py: Make --verbose mode more useful and default
ludvigm authored
609 warning("Redirected to: %s" % (redir_hostname))
4617f44 * s3cmd: Don't display download/upload completed message
ludvigm authored
610 return self.send_file(request, file, labels)
3ecbb36 2008-02-11 Michal Ludvig <michal@logix.cz>
ludvigm authored
611
8696851 * S3/S3.py: Re-upload when Amazon doesn't send ETag
ludvigm authored
612 # S3 from time to time doesn't send ETag back in a response :-(
613 # Force re-upload here.
614 if not response['headers'].has_key('etag'):
615 response['headers']['etag'] = ''
616
b67753b * S3/S3.py: Some errors during file upload were incorrectly
ludvigm authored
617 if response["status"] < 200 or response["status"] > 299:
b0b9791 * S3/Exceptions.py, S3/S3.py: Some HTTP_400 exceptions
ludvigm authored
618 try_retry = False
b67753b * S3/S3.py: Some errors during file upload were incorrectly
ludvigm authored
619 if response["status"] >= 500:
620 ## AWS internal error - retry
b0b9791 * S3/Exceptions.py, S3/S3.py: Some HTTP_400 exceptions
ludvigm authored
621 try_retry = True
622 elif response["status"] >= 400:
623 err = S3Error(response)
624 ## Retriable client error?
625 if err.code in [ 'BadDigest', 'OperationAborted', 'TokenRefreshRequired', 'RequestTimeout' ]:
626 try_retry = True
627
628 if try_retry:
b67753b * S3/S3.py: Some errors during file upload were incorrectly
ludvigm authored
629 if retries:
630 warning("Upload failed: %s (%s)" % (resource['uri'], S3Error(response)))
631 warning("Waiting %d sec..." % self._fail_wait(retries))
632 time.sleep(self._fail_wait(retries))
633 return self.send_file(request, file, labels, throttle, retries - 1)
634 else:
635 warning("Too many failures. Giving up on '%s'" % (file.name))
636 raise S3UploadError
b0b9791 * S3/Exceptions.py, S3/S3.py: Some HTTP_400 exceptions
ludvigm authored
637
b67753b * S3/S3.py: Some errors during file upload were incorrectly
ludvigm authored
638 ## Non-recoverable error
639 raise S3Error(response)
640
ab13aa8 * S3/S3.py: send_file() now computes MD5 sum of the file
ludvigm authored
641 debug("MD5 sums: computed=%s, received=%s" % (md5_computed, response["headers"]["etag"]))
642 if response["headers"]["etag"].strip('"\'') != md5_hash.hexdigest():
643 warning("MD5 Sums don't match!")
644 if retries:
20cb0e0 * s3cmd, S3/S3.py: Make --verbose mode more useful and default
ludvigm authored
645 warning("Retrying upload of %s" % (file.name))
4617f44 * s3cmd: Don't display download/upload completed message
ludvigm authored
646 return self.send_file(request, file, labels, throttle, retries - 1)
ab13aa8 * S3/S3.py: send_file() now computes MD5 sum of the file
ludvigm authored
647 else:
20cb0e0 * s3cmd, S3/S3.py: Make --verbose mode more useful and default
ludvigm authored
648 warning("Too many failures. Giving up on '%s'" % (file.name))
ab13aa8 * S3/S3.py: send_file() now computes MD5 sum of the file
ludvigm authored
649 raise S3UploadError
650
45d65cf Renamed s3py to s3cmd
mludvig authored
651 return response
652
4617f44 * s3cmd: Don't display download/upload completed message
ludvigm authored
653 def recv_file(self, request, stream, labels, start_position = 0, retries = _max_retries):
24f6313 * S3/S3.py: Re-sign requests before retrial to avoid
ludvigm authored
654 method_string, resource, headers = request.get_triplet()
e411362 * S3/Progress.py: Two progress meter implementations.
ludvigm authored
655 if self.config.progress_meter:
4617f44 * s3cmd: Don't display download/upload completed message
ludvigm authored
656 progress = self.config.progress_class(labels, 0)
e411362 * S3/Progress.py: Two progress meter implementations.
ludvigm authored
657 else:
658 info("Receiving file '%s', please wait..." % stream.name)
659 timestamp_start = time.time()
4f469e9 * S3/S3.py, s3cmd, S3/Config.py, s3cmd.1: Added --continue for
ludvigm authored
660 try:
661 conn = self.get_connection(resource['bucket'])
662 conn.connect()
663 conn.putrequest(method_string, self.format_uri(resource))
664 for header in headers.keys():
665 conn.putheader(header, str(headers[header]))
666 if start_position > 0:
667 debug("Requesting Range: %d .. end" % start_position)
668 conn.putheader("Range", "bytes=%d-" % start_position)
669 conn.endheaders()
670 response = {}
671 http_response = conn.getresponse()
672 response["status"] = http_response.status
673 response["reason"] = http_response.reason
674 response["headers"] = convertTupleListToDict(http_response.getheaders())
675 debug("Response: %s" % response)
676 except Exception, e:
677 if self.config.progress_meter:
678 progress.done("failed")
679 if retries:
680 warning("Retrying failed request: %s (%s)" % (resource['uri'], e))
681 warning("Waiting %d sec..." % self._fail_wait(retries))
682 time.sleep(self._fail_wait(retries))
683 # Connection error -> same throttle value
4617f44 * s3cmd: Don't display download/upload completed message
ludvigm authored
684 return self.recv_file(request, stream, labels, start_position, retries - 1)
4f469e9 * S3/S3.py, s3cmd, S3/Config.py, s3cmd.1: Added --continue for
ludvigm authored
685 else:
686 raise S3DownloadError("Download failed for: %s" % resource['uri'])
3ecbb36 2008-02-11 Michal Ludvig <michal@logix.cz>
ludvigm authored
687
688 if response["status"] == 307:
689 ## RedirectPermanent
690 response['data'] = http_response.read()
691 redir_bucket = getTextFromXml(response['data'], ".//Bucket")
692 redir_hostname = getTextFromXml(response['data'], ".//Endpoint")
693 self.set_hostname(redir_bucket, redir_hostname)
20cb0e0 * s3cmd, S3/S3.py: Make --verbose mode more useful and default
ludvigm authored
694 warning("Redirected to: %s" % (redir_hostname))
4617f44 * s3cmd: Don't display download/upload completed message
ludvigm authored
695 return self.recv_file(request, stream, labels)
3ecbb36 2008-02-11 Michal Ludvig <michal@logix.cz>
ludvigm authored
696
45d65cf Renamed s3py to s3cmd
mludvig authored
697 if response["status"] < 200 or response["status"] > 299:
698 raise S3Error(response)
699
4f469e9 * S3/S3.py, s3cmd, S3/Config.py, s3cmd.1: Added --continue for
ludvigm authored
700 if start_position == 0:
701 # Only compute MD5 on the fly if we're downloading from beginning
702 # Otherwise we'd get a nonsense.
4ffe940 * S3/S3.py, S3/Utils.py: Use 'hashlib' instead of md5 and sha
w_tell authored
703 md5_hash = md5()
4f469e9 * S3/S3.py, s3cmd, S3/Config.py, s3cmd.1: Added --continue for
ludvigm authored
704 size_left = int(response["headers"]["content-length"])
705 size_total = start_position + size_left
706 current_position = start_position
707
e411362 * S3/Progress.py: Two progress meter implementations.
ludvigm authored
708 if self.config.progress_meter:
709 progress.total_size = size_total
4f469e9 * S3/S3.py, s3cmd, S3/Config.py, s3cmd.1: Added --continue for
ludvigm authored
710 progress.initial_position = current_position
711 progress.current_position = current_position
712
713 try:
714 while (current_position < size_total):
715 this_chunk = size_left > self.config.recv_chunk and self.config.recv_chunk or size_left
716 data = http_response.read(this_chunk)
717 stream.write(data)
718 if start_position == 0:
719 md5_hash.update(data)
720 current_position += len(data)
721 ## Call progress meter from here...
722 if self.config.progress_meter:
723 progress.update(delta_position = len(data))
724 conn.close()
725 except Exception, e:
e411362 * S3/Progress.py: Two progress meter implementations.
ludvigm authored
726 if self.config.progress_meter:
4f469e9 * S3/S3.py, s3cmd, S3/Config.py, s3cmd.1: Added --continue for
ludvigm authored
727 progress.done("failed")
728 if retries:
729 warning("Retrying failed request: %s (%s)" % (resource['uri'], e))
730 warning("Waiting %d sec..." % self._fail_wait(retries))
731 time.sleep(self._fail_wait(retries))
732 # Connection error -> same throttle value
4617f44 * s3cmd: Don't display download/upload completed message
ludvigm authored
733 return self.recv_file(request, stream, labels, current_position, retries - 1)
e411362 * S3/Progress.py: Two progress meter implementations.
ludvigm authored
734 else:
4f469e9 * S3/S3.py, s3cmd, S3/Config.py, s3cmd.1: Added --continue for
ludvigm authored
735 raise S3DownloadError("Download failed for: %s" % resource['uri'])
736
737 stream.flush()
b1eaa42 * s3cmd, S3/S3.py, S3/Utils.py: Throttle upload speed and retry
ludvigm authored
738 timestamp_end = time.time()
4f469e9 * S3/S3.py, s3cmd, S3/Config.py, s3cmd.1: Added --continue for
ludvigm authored
739
fd79525 * s3cmd, s3cmd.1, S3/S3.py: Display or don't display progress meter
ludvigm authored
740 if self.config.progress_meter:
741 ## The above stream.flush() may take some time -> update() progress meter
742 ## to correct the average speed. Otherwise people will complain that
743 ## 'progress' and response["speed"] are inconsistent ;-)
744 progress.update()
745 progress.done("done")
746
4f469e9 * S3/S3.py, s3cmd, S3/Config.py, s3cmd.1: Added --continue for
ludvigm authored
747 if start_position == 0:
748 # Only compute MD5 on the fly if we were downloading from the beginning
749 response["md5"] = md5_hash.hexdigest()
750 else:
751 # Otherwise try to compute MD5 of the output file
752 try:
753 response["md5"] = hash_file_md5(stream.name)
754 except IOError, e:
755 if e.errno != errno.ENOENT:
756 warning("Unable to open file: %s: %s" % (stream.name, e))
757 warning("Unable to verify MD5. Assume it matches.")
758 response["md5"] = response["headers"]["etag"]
759
45d65cf Renamed s3py to s3cmd
mludvig authored
760 response["md5match"] = response["headers"]["etag"].find(response["md5"]) >= 0
b1eaa42 * s3cmd, S3/S3.py, S3/Utils.py: Throttle upload speed and retry
ludvigm authored
761 response["elapsed"] = timestamp_end - timestamp_start
4f469e9 * S3/S3.py, s3cmd, S3/Config.py, s3cmd.1: Added --continue for
ludvigm authored
762 response["size"] = current_position
2644cd7 * S3/S3.py: Don't run into ZeroDivisionError when speed counter
ludvigm authored
763 response["speed"] = response["elapsed"] and float(response["size"]) / response["elapsed"] or float(-1)
4f469e9 * S3/S3.py, s3cmd, S3/Config.py, s3cmd.1: Added --continue for
ludvigm authored
764 if response["size"] != start_position + long(response["headers"]["content-length"]):
a580bd1 - New feature - allow "get" to stdout
mludvig authored
765 warning("Reported size (%s) does not match received size (%s)" % (
4f469e9 * S3/S3.py, s3cmd, S3/Config.py, s3cmd.1: Added --continue for
ludvigm authored
766 start_position + response["headers"]["content-length"], response["size"]))
45d65cf Renamed s3py to s3cmd
mludvig authored
767 debug("ReceiveFile: Computed MD5 = %s" % response["md5"])
768 if not response["md5match"]:
769 warning("MD5 signatures do not match: computed=%s, received=%s" % (
770 response["md5"], response["headers"]["etag"]))
771 return response
5c3554d * s3cmd, S3/AccessLog.py, ...: Added [accesslog] command.
ludvigm authored
772 __all__.append("S3")
Something went wrong with that request. Please try again.