Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Newer
Older
100755 1962 lines (1682 sloc) 90.797 kb
af984ba @canadianveggie Fixing bug 3091912 - KeyError when copying multiple keys
canadianveggie authored
1 #!/usr/bin/python
3cc025a @mludvig Renamed s3.py to s3cmd
mludvig authored
2
3 ## Amazon S3 manager
4 ## Author: Michal Ludvig <michal@logix.cz>
5 ## http://www.logix.cz/michal
6 ## License: GPL Version 2
7
8 import sys
5c805fd @mludvig * s3cmd: Check Python version >= 2.4 as soon as possible.
mludvig authored
9
10 if float("%d.%d" %(sys.version_info[0], sys.version_info[1])) < 2.4:
d439efb @mludvig ATTENTION -- Mega WhiteSpace conversion !!!
mludvig authored
11 sys.stderr.write("ERROR: Python 2.4 or higher required, sorry.\n")
12 sys.exit(1)
5c805fd @mludvig * s3cmd: Check Python version >= 2.4 as soon as possible.
mludvig authored
13
3cc025a @mludvig Renamed s3.py to s3cmd
mludvig authored
14 import logging
15 import time
8a4a98b @mludvig 2007-07-05 Michal Ludvig <michal@logix.cz>
mludvig authored
16 import os
17 import re
ac9940e @mludvig 2007-09-25 Michal Ludvig <michal@logix.cz>
mludvig authored
18 import errno
2d7d554 @mludvig * s3cmd, s3cmd.1: Added GLOB (shell-style wildcard) exclude, renamed
mludvig authored
19 import glob
4a52baa @mludvig * s3cmd: Wrapped all execution in a try/except block
mludvig authored
20 import traceback
4da602a @mludvig * s3cmd: Always output UTF-8, even on output redirects.
mludvig authored
21 import codecs
315e527 @mludvig * s3cmd: Replace unknown Unicode characters with '?'
mludvig authored
22 import locale
e3afa96 @mludvig * s3cmd: Rewritten gpg_command() to use subprocess.Popen()
mludvig authored
23 import subprocess
3c07424 @mludvig * s3cmd: New [fixbucket] command for fixing invalid object
mludvig authored
24 import htmlentitydefs
0222ce4 @mludvig * s3cmd: Set 10s socket timeout for read()/write().
mludvig authored
25 import socket
ddb5ef9 @mdomsch handle remote->local transfers with local hardlink/copy if possible
authored
26 import shutil
3cc025a @mludvig Renamed s3.py to s3cmd
mludvig authored
27
9b7618a @mludvig Major rework of Config class:
mludvig authored
28 from copy import copy
f4555c3 @mludvig - Use prefix for listings if specified.
mludvig authored
29 from optparse import OptionParser, Option, OptionValueError, IndentedHelpFormatter
3cc025a @mludvig Renamed s3.py to s3cmd
mludvig authored
30 from logging import debug, info, warning, error
49731b4 @mludvig 2007-06-17 Michal Ludvig <michal@logix.cz>
mludvig authored
31 from distutils.spawn import find_executable
3cc025a @mludvig Renamed s3.py to s3cmd
mludvig authored
32
33 def output(message):
d439efb @mludvig ATTENTION -- Mega WhiteSpace conversion !!!
mludvig authored
34 sys.stdout.write(message + "\n")
3cc025a @mludvig Renamed s3.py to s3cmd
mludvig authored
35
7c07fd6 @mludvig 2007-08-19 Michal Ludvig <michal@logix.cz>
mludvig authored
36 def check_args_type(args, type, verbose_type):
d439efb @mludvig ATTENTION -- Mega WhiteSpace conversion !!!
mludvig authored
37 for arg in args:
38 if S3Uri(arg).type != type:
39 raise ParameterError("Expecting %s instead of '%s'" % (verbose_type, arg))
7c07fd6 @mludvig 2007-08-19 Michal Ludvig <michal@logix.cz>
mludvig authored
40
b96ddeb @mludvig 2007-05-26 Michal Ludvig <michal@logix.cz>
mludvig authored
41 def cmd_du(args):
d439efb @mludvig ATTENTION -- Mega WhiteSpace conversion !!!
mludvig authored
42 s3 = S3(Config())
43 if len(args) > 0:
44 uri = S3Uri(args[0])
45 if uri.type == "s3" and uri.has_bucket():
46 subcmd_bucket_usage(s3, uri)
47 return
48 subcmd_bucket_usage_all(s3)
b96ddeb @mludvig 2007-05-26 Michal Ludvig <michal@logix.cz>
mludvig authored
49
50 def subcmd_bucket_usage_all(s3):
d439efb @mludvig ATTENTION -- Mega WhiteSpace conversion !!!
mludvig authored
51 response = s3.list_all_buckets()
52
53 buckets_size = 0
54 for bucket in response["list"]:
55 size = subcmd_bucket_usage(s3, S3Uri("s3://" + bucket["Name"]))
56 if size != None:
57 buckets_size += size
58 total_size, size_coeff = formatSize(buckets_size, Config().human_readable_sizes)
59 total_size_str = str(total_size) + size_coeff
60 output(u"".rjust(8, "-"))
61 output(u"%s Total" % (total_size_str.ljust(8)))
b96ddeb @mludvig 2007-05-26 Michal Ludvig <michal@logix.cz>
mludvig authored
62
63 def subcmd_bucket_usage(s3, uri):
d439efb @mludvig ATTENTION -- Mega WhiteSpace conversion !!!
mludvig authored
64 bucket = uri.bucket()
65 object = uri.object()
66
67 if object.endswith('*'):
68 object = object[:-1]
69 try:
70 response = s3.bucket_list(bucket, prefix = object, recursive = True)
71 except S3Error, e:
72 if S3.codes.has_key(e.info["Code"]):
73 error(S3.codes[e.info["Code"]] % bucket)
74 return
75 else:
76 raise
77 bucket_size = 0
78 for object in response["list"]:
79 size, size_coeff = formatSize(object["Size"], False)
80 bucket_size += size
81 total_size, size_coeff = formatSize(bucket_size, Config().human_readable_sizes)
82 total_size_str = str(total_size) + size_coeff
83 output(u"%s %s" % (total_size_str.ljust(8), uri))
84 return bucket_size
b96ddeb @mludvig 2007-05-26 Michal Ludvig <michal@logix.cz>
mludvig authored
85
9081133 @mludvig - Merged list-buckets and bucket-list-objects operations into
mludvig authored
86 def cmd_ls(args):
d439efb @mludvig ATTENTION -- Mega WhiteSpace conversion !!!
mludvig authored
87 s3 = S3(Config())
88 if len(args) > 0:
89 uri = S3Uri(args[0])
90 if uri.type == "s3" and uri.has_bucket():
91 subcmd_bucket_list(s3, uri)
92 return
93 subcmd_buckets_list_all(s3)
3cc025a @mludvig Renamed s3.py to s3cmd
mludvig authored
94
95 def cmd_buckets_list_all_all(args):
d439efb @mludvig ATTENTION -- Mega WhiteSpace conversion !!!
mludvig authored
96 s3 = S3(Config())
b819c70 @mludvig - Converted all users of parse_uri to S3Uri class API
mludvig authored
97
d439efb @mludvig ATTENTION -- Mega WhiteSpace conversion !!!
mludvig authored
98 response = s3.list_all_buckets()
3cc025a @mludvig Renamed s3.py to s3cmd
mludvig authored
99
d439efb @mludvig ATTENTION -- Mega WhiteSpace conversion !!!
mludvig authored
100 for bucket in response["list"]:
101 subcmd_bucket_list(s3, S3Uri("s3://" + bucket["Name"]))
102 output(u"")
3cc025a @mludvig Renamed s3.py to s3cmd
mludvig authored
103
104
b819c70 @mludvig - Converted all users of parse_uri to S3Uri class API
mludvig authored
105 def subcmd_buckets_list_all(s3):
d439efb @mludvig ATTENTION -- Mega WhiteSpace conversion !!!
mludvig authored
106 response = s3.list_all_buckets()
107 for bucket in response["list"]:
108 output(u"%s s3://%s" % (
109 formatDateTime(bucket["CreationDate"]),
110 bucket["Name"],
111 ))
b819c70 @mludvig - Converted all users of parse_uri to S3Uri class API
mludvig authored
112
113 def subcmd_bucket_list(s3, uri):
d439efb @mludvig ATTENTION -- Mega WhiteSpace conversion !!!
mludvig authored
114 bucket = uri.bucket()
115 prefix = uri.object()
116
117 debug(u"Bucket 's3://%s':" % bucket)
118 if prefix.endswith('*'):
119 prefix = prefix[:-1]
120 try:
121 response = s3.bucket_list(bucket, prefix = prefix)
122 except S3Error, e:
123 if S3.codes.has_key(e.info["Code"]):
124 error(S3.codes[e.info["Code"]] % bucket)
125 return
126 else:
127 raise
128
129 if cfg.list_md5:
130 format_string = u"%(timestamp)16s %(size)9s%(coeff)1s %(md5)32s %(uri)s"
131 else:
132 format_string = u"%(timestamp)16s %(size)9s%(coeff)1s %(uri)s"
133
134 for prefix in response['common_prefixes']:
135 output(format_string % {
136 "timestamp": "",
137 "size": "DIR",
138 "coeff": "",
139 "md5": "",
140 "uri": uri.compose_uri(bucket, prefix["Prefix"])})
141
142 for object in response["list"]:
143 size, size_coeff = formatSize(object["Size"], Config().human_readable_sizes)
144 output(format_string % {
145 "timestamp": formatDateTime(object["LastModified"]),
146 "size" : str(size),
147 "coeff": size_coeff,
148 "md5" : object['ETag'].strip('"'),
149 "uri": uri.compose_uri(bucket, object["Key"]),
150 })
3cc025a @mludvig Renamed s3.py to s3cmd
mludvig authored
151
152 def cmd_bucket_create(args):
d439efb @mludvig ATTENTION -- Mega WhiteSpace conversion !!!
mludvig authored
153 s3 = S3(Config())
154 for arg in args:
155 uri = S3Uri(arg)
156 if not uri.type == "s3" or not uri.has_bucket() or uri.has_object():
157 raise ParameterError("Expecting S3 URI with just the bucket name set instead of '%s'" % arg)
158 try:
159 response = s3.bucket_create(uri.bucket(), cfg.bucket_location)
160 output(u"Bucket '%s' created" % uri.uri())
161 except S3Error, e:
162 if S3.codes.has_key(e.info["Code"]):
163 error(S3.codes[e.info["Code"]] % uri.bucket())
164 return
165 else:
166 raise
3cc025a @mludvig Renamed s3.py to s3cmd
mludvig authored
167
3bf7d0c @mludvig Improved WebSite support
mludvig authored
168 def cmd_website_info(args):
d439efb @mludvig ATTENTION -- Mega WhiteSpace conversion !!!
mludvig authored
169 s3 = S3(Config())
170 for arg in args:
171 uri = S3Uri(arg)
172 if not uri.type == "s3" or not uri.has_bucket() or uri.has_object():
173 raise ParameterError("Expecting S3 URI with just the bucket name set instead of '%s'" % arg)
174 try:
175 response = s3.website_info(uri, cfg.bucket_location)
176 if response:
177 output(u"Bucket %s: Website configuration" % uri.uri())
178 output(u"Website endpoint: %s" % response['website_endpoint'])
179 output(u"Index document: %s" % response['index_document'])
180 output(u"Error document: %s" % response['error_document'])
181 else:
182 output(u"Bucket %s: Unable to receive website configuration." % (uri.uri()))
183 except S3Error, e:
184 if S3.codes.has_key(e.info["Code"]):
185 error(S3.codes[e.info["Code"]] % uri.bucket())
186 return
187 else:
188 raise
cde72d4 Added support for "Static Websites"
Jens Braeuer authored
189
190 def cmd_website_create(args):
d439efb @mludvig ATTENTION -- Mega WhiteSpace conversion !!!
mludvig authored
191 s3 = S3(Config())
192 for arg in args:
193 uri = S3Uri(arg)
194 if not uri.type == "s3" or not uri.has_bucket() or uri.has_object():
195 raise ParameterError("Expecting S3 URI with just the bucket name set instead of '%s'" % arg)
196 try:
197 response = s3.website_create(uri, cfg.bucket_location)
198 output(u"Bucket '%s': website configuration created." % (uri.uri()))
199 except S3Error, e:
200 if S3.codes.has_key(e.info["Code"]):
201 error(S3.codes[e.info["Code"]] % uri.bucket())
202 return
203 else:
204 raise
cde72d4 Added support for "Static Websites"
Jens Braeuer authored
205
206 def cmd_website_delete(args):
d439efb @mludvig ATTENTION -- Mega WhiteSpace conversion !!!
mludvig authored
207 s3 = S3(Config())
208 for arg in args:
209 uri = S3Uri(arg)
210 if not uri.type == "s3" or not uri.has_bucket() or uri.has_object():
211 raise ParameterError("Expecting S3 URI with just the bucket name set instead of '%s'" % arg)
212 try:
213 response = s3.website_delete(uri, cfg.bucket_location)
214 output(u"Bucket '%s': website configuration deleted." % (uri.uri()))
215 except S3Error, e:
216 if S3.codes.has_key(e.info["Code"]):
217 error(S3.codes[e.info["Code"]] % uri.bucket())
218 return
219 else:
220 raise
cde72d4 Added support for "Static Websites"
Jens Braeuer authored
221
3cc025a @mludvig Renamed s3.py to s3cmd
mludvig authored
222 def cmd_bucket_delete(args):
d439efb @mludvig ATTENTION -- Mega WhiteSpace conversion !!!
mludvig authored
223 def _bucket_delete_one(uri):
224 try:
225 response = s3.bucket_delete(uri.bucket())
226 except S3Error, e:
227 if e.info['Code'] == 'BucketNotEmpty' and (cfg.force or cfg.recursive):
228 warning(u"Bucket is not empty. Removing all the objects from it first. This may take some time...")
229 subcmd_object_del_uri(uri.uri(), recursive = True)
230 return _bucket_delete_one(uri)
231 elif S3.codes.has_key(e.info["Code"]):
232 error(S3.codes[e.info["Code"]] % uri.bucket())
233 return
234 else:
235 raise
236
237 s3 = S3(Config())
238 for arg in args:
239 uri = S3Uri(arg)
240 if not uri.type == "s3" or not uri.has_bucket() or uri.has_object():
241 raise ParameterError("Expecting S3 URI with just the bucket name set instead of '%s'" % arg)
242 _bucket_delete_one(uri)
243 output(u"Bucket '%s' removed" % uri.uri())
f4555c3 @mludvig - Use prefix for listings if specified.
mludvig authored
244
3cc025a @mludvig Renamed s3.py to s3cmd
mludvig authored
245 def cmd_object_put(args):
d439efb @mludvig ATTENTION -- Mega WhiteSpace conversion !!!
mludvig authored
246 cfg = Config()
247 s3 = S3(cfg)
248
249 if len(args) == 0:
250 raise ParameterError("Nothing to upload. Expecting a local file or directory and a S3 URI destination.")
251
252 ## Normalize URI to convert s3://bkt to s3://bkt/ (trailing slash)
253 destination_base_uri = S3Uri(args.pop())
254 if destination_base_uri.type != 's3':
255 raise ParameterError("Destination must be S3Uri. Got: %s" % destination_base_uri)
256 destination_base = str(destination_base_uri)
257
258 if len(args) == 0:
259 raise ParameterError("Nothing to upload. Expecting a local file or directory.")
260
261 local_list, single_file_local = fetch_local_list(args)
262
263 local_list, exclude_list = filter_exclude_include(local_list)
264
265 local_count = len(local_list)
266
267 info(u"Summary: %d local files to upload" % local_count)
268
269 if local_count > 0:
270 if not destination_base.endswith("/"):
271 if not single_file_local:
272 raise ParameterError("Destination S3 URI must end with '/' (ie must refer to a directory on the remote side).")
273 local_list[local_list.keys()[0]]['remote_uri'] = unicodise(destination_base)
274 else:
275 for key in local_list:
276 local_list[key]['remote_uri'] = unicodise(destination_base + key)
277
278 if cfg.dry_run:
279 for key in exclude_list:
280 output(u"exclude: %s" % unicodise(key))
281 for key in local_list:
282 output(u"upload: %s -> %s" % (local_list[key]['full_name_unicode'], local_list[key]['remote_uri']))
283
284 warning(u"Exitting now because of --dry-run")
285 return
286
287 seq = 0
288 for key in local_list:
289 seq += 1
290
291 uri_final = S3Uri(local_list[key]['remote_uri'])
292
293 extra_headers = copy(cfg.extra_headers)
294 full_name_orig = local_list[key]['full_name']
295 full_name = full_name_orig
296 seq_label = "[%d of %d]" % (seq, local_count)
297 if Config().encrypt:
298 exitcode, full_name, extra_headers["x-amz-meta-s3tools-gpgenc"] = gpg_encrypt(full_name_orig)
299 try:
300 response = s3.object_put(full_name, uri_final, extra_headers, extra_label = seq_label)
301 except S3UploadError, e:
302 error(u"Upload of '%s' failed too many times. Skipping that file." % full_name_orig)
303 continue
304 except InvalidFileError, e:
305 warning(u"File can not be uploaded: %s" % e)
306 continue
307 speed_fmt = formatSize(response["speed"], human_readable = True, floating_point = True)
308 if not Config().progress_meter:
309 output(u"File '%s' stored as '%s' (%d bytes in %0.1f seconds, %0.2f %sB/s) %s" %
310 (unicodise(full_name_orig), uri_final, response["size"], response["elapsed"],
311 speed_fmt[0], speed_fmt[1], seq_label))
312 if Config().acl_public:
313 output(u"Public URL of the object is: %s" %
314 (uri_final.public_url()))
315 if Config().encrypt and full_name != full_name_orig:
316 debug(u"Removing temporary encrypted file: %s" % unicodise(full_name))
317 os.remove(full_name)
3cc025a @mludvig Renamed s3.py to s3cmd
mludvig authored
318
319 def cmd_object_get(args):
d439efb @mludvig ATTENTION -- Mega WhiteSpace conversion !!!
mludvig authored
320 cfg = Config()
321 s3 = S3(cfg)
322
323 ## Check arguments:
324 ## if not --recursive:
325 ## - first N arguments must be S3Uri
326 ## - if the last one is S3 make current dir the destination_base
327 ## - if the last one is a directory:
328 ## - take all 'basenames' of the remote objects and
329 ## make the destination name be 'destination_base'+'basename'
330 ## - if the last one is a file or not existing:
331 ## - if the number of sources (N, above) == 1 treat it
332 ## as a filename and save the object there.
333 ## - if there's more sources -> Error
334 ## if --recursive:
335 ## - first N arguments must be S3Uri
336 ## - for each Uri get a list of remote objects with that Uri as a prefix
337 ## - apply exclude/include rules
338 ## - each list item will have MD5sum, Timestamp and pointer to S3Uri
339 ## used as a prefix.
2320b45 @ohhorob Allow stdout as destination when receiving multiple remote files
ohhorob authored
340 ## - the last arg may be '-' (stdout)
d439efb @mludvig ATTENTION -- Mega WhiteSpace conversion !!!
mludvig authored
341 ## - the last arg may be a local directory - destination_base
342 ## - if the last one is S3 make current dir the destination_base
343 ## - if the last one doesn't exist check remote list:
344 ## - if there is only one item and its_prefix==its_name
345 ## download that item to the name given in last arg.
346 ## - if there are more remote items use the last arg as a destination_base
347 ## and try to create the directory (incl. all parents).
348 ##
349 ## In both cases we end up with a list mapping remote object names (keys) to local file names.
350
351 ## Each item will be a dict with the following attributes
352 # {'remote_uri', 'local_filename'}
353 download_list = []
354
355 if len(args) == 0:
356 raise ParameterError("Nothing to download. Expecting S3 URI.")
357
358 if S3Uri(args[-1]).type == 'file':
359 destination_base = args.pop()
360 else:
361 destination_base = "."
362
363 if len(args) == 0:
364 raise ParameterError("Nothing to download. Expecting S3 URI.")
365
366 remote_list = fetch_remote_list(args, require_attribs = False)
367 remote_list, exclude_list = filter_exclude_include(remote_list)
368
369 remote_count = len(remote_list)
370
371 info(u"Summary: %d remote files to download" % remote_count)
372
373 if remote_count > 0:
2320b45 @ohhorob Allow stdout as destination when receiving multiple remote files
ohhorob authored
374 if destination_base == "-":
375 ## stdout is ok for multiple remote files!
376 for key in remote_list:
377 remote_list[key]['local_filename'] = "-"
378 elif not os.path.isdir(destination_base):
379 ## We were either given a file name (existing or not)
d439efb @mludvig ATTENTION -- Mega WhiteSpace conversion !!!
mludvig authored
380 if remote_count > 1:
2320b45 @ohhorob Allow stdout as destination when receiving multiple remote files
ohhorob authored
381 raise ParameterError("Destination must be a directory or stdout when downloading multiple sources.")
d439efb @mludvig ATTENTION -- Mega WhiteSpace conversion !!!
mludvig authored
382 remote_list[remote_list.keys()[0]]['local_filename'] = deunicodise(destination_base)
383 elif os.path.isdir(destination_base):
384 if destination_base[-1] != os.path.sep:
385 destination_base += os.path.sep
386 for key in remote_list:
387 remote_list[key]['local_filename'] = destination_base + key
388 else:
389 raise InternalError("WTF? Is it a dir or not? -- %s" % destination_base)
390
391 if cfg.dry_run:
392 for key in exclude_list:
393 output(u"exclude: %s" % unicodise(key))
394 for key in remote_list:
395 output(u"download: %s -> %s" % (remote_list[key]['object_uri_str'], remote_list[key]['local_filename']))
396
397 warning(u"Exitting now because of --dry-run")
398 return
399
400 seq = 0
401 for key in remote_list:
402 seq += 1
403 item = remote_list[key]
404 uri = S3Uri(item['object_uri_str'])
405 ## Encode / Decode destination with "replace" to make sure it's compatible with current encoding
406 destination = unicodise_safe(item['local_filename'])
407 seq_label = "[%d of %d]" % (seq, remote_count)
408
409 start_position = 0
410
411 if destination == "-":
412 ## stdout
413 dst_stream = sys.__stdout__
414 else:
415 ## File
416 try:
417 file_exists = os.path.exists(destination)
418 try:
419 dst_stream = open(destination, "ab")
420 except IOError, e:
421 if e.errno == errno.ENOENT:
422 basename = destination[:destination.rindex(os.path.sep)]
423 info(u"Creating directory: %s" % basename)
424 os.makedirs(basename)
425 dst_stream = open(destination, "ab")
426 else:
427 raise
428 if file_exists:
429 if Config().get_continue:
430 start_position = dst_stream.tell()
431 elif Config().force:
432 start_position = 0L
433 dst_stream.seek(0L)
434 dst_stream.truncate()
435 elif Config().skip_existing:
436 info(u"Skipping over existing file: %s" % (destination))
437 continue
438 else:
439 dst_stream.close()
440 raise ParameterError(u"File %s already exists. Use either of --force / --continue / --skip-existing or give it a new name." % destination)
441 except IOError, e:
442 error(u"Skipping %s: %s" % (destination, e.strerror))
443 continue
444 response = s3.object_get(uri, dst_stream, start_position = start_position, extra_label = seq_label)
445 if response["headers"].has_key("x-amz-meta-s3tools-gpgenc"):
446 gpg_decrypt(destination, response["headers"]["x-amz-meta-s3tools-gpgenc"])
447 response["size"] = os.stat(destination)[6]
448 if not Config().progress_meter and destination != "-":
449 speed_fmt = formatSize(response["speed"], human_readable = True, floating_point = True)
450 output(u"File %s saved as '%s' (%d bytes in %0.1f seconds, %0.2f %sB/s)" %
451 (uri, destination, response["size"], response["elapsed"], speed_fmt[0], speed_fmt[1]))
3cc025a @mludvig Renamed s3.py to s3cmd
mludvig authored
452
453 def cmd_object_del(args):
d439efb @mludvig ATTENTION -- Mega WhiteSpace conversion !!!
mludvig authored
454 for uri_str in args:
455 uri = S3Uri(uri_str)
456 if uri.type != "s3":
457 raise ParameterError("Expecting S3 URI instead of '%s'" % uri_str)
458 if not uri.has_object():
459 if Config().recursive and not Config().force:
460 raise ParameterError("Please use --force to delete ALL contents of %s" % uri_str)
461 elif not Config().recursive:
462 raise ParameterError("File name required, not only the bucket name. Alternatively use --recursive")
463 subcmd_object_del_uri(uri_str)
1ae39a8 @mludvig * s3cmd, run-test.py, TODO, NEWS: Added --dry-run
mludvig authored
464
465 def subcmd_object_del_uri(uri_str, recursive = None):
d439efb @mludvig ATTENTION -- Mega WhiteSpace conversion !!!
mludvig authored
466 s3 = S3(cfg)
7406fc6 @mludvig * s3cmd, S3/Config.py: [rb] Allow removal of non-empty buckets
mludvig authored
467
d439efb @mludvig ATTENTION -- Mega WhiteSpace conversion !!!
mludvig authored
468 if recursive is None:
469 recursive = cfg.recursive
1ae39a8 @mludvig * s3cmd, run-test.py, TODO, NEWS: Added --dry-run
mludvig authored
470
d439efb @mludvig ATTENTION -- Mega WhiteSpace conversion !!!
mludvig authored
471 remote_list = fetch_remote_list(uri_str, require_attribs = False, recursive = recursive)
472 remote_list, exclude_list = filter_exclude_include(remote_list)
1ae39a8 @mludvig * s3cmd, run-test.py, TODO, NEWS: Added --dry-run
mludvig authored
473
d439efb @mludvig ATTENTION -- Mega WhiteSpace conversion !!!
mludvig authored
474 remote_count = len(remote_list)
1ae39a8 @mludvig * s3cmd, run-test.py, TODO, NEWS: Added --dry-run
mludvig authored
475
d439efb @mludvig ATTENTION -- Mega WhiteSpace conversion !!!
mludvig authored
476 info(u"Summary: %d remote files to delete" % remote_count)
1ae39a8 @mludvig * s3cmd, run-test.py, TODO, NEWS: Added --dry-run
mludvig authored
477
d439efb @mludvig ATTENTION -- Mega WhiteSpace conversion !!!
mludvig authored
478 if cfg.dry_run:
479 for key in exclude_list:
480 output(u"exclude: %s" % unicodise(key))
481 for key in remote_list:
482 output(u"delete: %s" % remote_list[key]['object_uri_str'])
1ae39a8 @mludvig * s3cmd, run-test.py, TODO, NEWS: Added --dry-run
mludvig authored
483
d439efb @mludvig ATTENTION -- Mega WhiteSpace conversion !!!
mludvig authored
484 warning(u"Exitting now because of --dry-run")
485 return
1ae39a8 @mludvig * s3cmd, run-test.py, TODO, NEWS: Added --dry-run
mludvig authored
486
d439efb @mludvig ATTENTION -- Mega WhiteSpace conversion !!!
mludvig authored
487 for key in remote_list:
488 item = remote_list[key]
489 response = s3.object_delete(S3Uri(item['object_uri_str']))
490 output(u"File %s deleted" % item['object_uri_str'])
3cc025a @mludvig Renamed s3.py to s3cmd
mludvig authored
491
e0b946c @mludvig * s3cmd: Support for recursive [cp] and [mv], including
mludvig authored
492 def subcmd_cp_mv(args, process_fce, action_str, message):
d439efb @mludvig ATTENTION -- Mega WhiteSpace conversion !!!
mludvig authored
493 if len(args) < 2:
494 raise ParameterError("Expecting two or more S3 URIs for " + action_str)
495 dst_base_uri = S3Uri(args.pop())
496 if dst_base_uri.type != "s3":
497 raise ParameterError("Destination must be S3 URI. To download a file use 'get' or 'sync'.")
498 destination_base = dst_base_uri.uri()
499
500 remote_list = fetch_remote_list(args, require_attribs = False)
501 remote_list, exclude_list = filter_exclude_include(remote_list)
502
503 remote_count = len(remote_list)
504
505 info(u"Summary: %d remote files to %s" % (remote_count, action_str))
506
507 if cfg.recursive:
508 if not destination_base.endswith("/"):
509 destination_base += "/"
510 for key in remote_list:
511 remote_list[key]['dest_name'] = destination_base + key
512 else:
af984ba @canadianveggie Fixing bug 3091912 - KeyError when copying multiple keys
canadianveggie authored
513 for key in remote_list:
514 if destination_base.endswith("/"):
515 remote_list[key]['dest_name'] = destination_base + key
516 else:
517 remote_list[key]['dest_name'] = destination_base
d439efb @mludvig ATTENTION -- Mega WhiteSpace conversion !!!
mludvig authored
518
519 if cfg.dry_run:
520 for key in exclude_list:
521 output(u"exclude: %s" % unicodise(key))
522 for key in remote_list:
523 output(u"%s: %s -> %s" % (action_str, remote_list[key]['object_uri_str'], remote_list[key]['dest_name']))
524
525 warning(u"Exitting now because of --dry-run")
526 return
527
528 seq = 0
529 for key in remote_list:
530 seq += 1
531 seq_label = "[%d of %d]" % (seq, remote_count)
532
533 item = remote_list[key]
534 src_uri = S3Uri(item['object_uri_str'])
535 dst_uri = S3Uri(item['dest_name'])
536
537 extra_headers = copy(cfg.extra_headers)
538 response = process_fce(src_uri, dst_uri, extra_headers)
539 output(message % { "src" : src_uri, "dst" : dst_uri })
540 if Config().acl_public:
541 info(u"Public URL is: %s" % dst_uri.public_url())
7d0ac8e @mludvig * s3cmd: Support for 'cp' command.
mludvig authored
542
7d61be8 @mludvig * s3cmd, S3/S3.py, NEWS: "s3cmd mv" for moving objects
mludvig authored
543 def cmd_cp(args):
d439efb @mludvig ATTENTION -- Mega WhiteSpace conversion !!!
mludvig authored
544 s3 = S3(Config())
545 subcmd_cp_mv(args, s3.object_copy, "copy", "File %(src)s copied to %(dst)s")
7d61be8 @mludvig * s3cmd, S3/S3.py, NEWS: "s3cmd mv" for moving objects
mludvig authored
546
547 def cmd_mv(args):
d439efb @mludvig ATTENTION -- Mega WhiteSpace conversion !!!
mludvig authored
548 s3 = S3(Config())
549 subcmd_cp_mv(args, s3.object_move, "move", "File %(src)s moved to %(dst)s")
7d61be8 @mludvig * s3cmd, S3/S3.py, NEWS: "s3cmd mv" for moving objects
mludvig authored
550
e5c6f6c @mludvig 2008-02-27 Michal Ludvig <michal@logix.cz>
mludvig authored
551 def cmd_info(args):
d439efb @mludvig ATTENTION -- Mega WhiteSpace conversion !!!
mludvig authored
552 s3 = S3(Config())
553
554 while (len(args)):
555 uri_arg = args.pop(0)
556 uri = S3Uri(uri_arg)
557 if uri.type != "s3" or not uri.has_bucket():
558 raise ParameterError("Expecting S3 URI instead of '%s'" % uri_arg)
559
560 try:
561 if uri.has_object():
562 info = s3.object_info(uri)
563 output(u"%s (object):" % uri.uri())
564 output(u" File size: %s" % info['headers']['content-length'])
565 output(u" Last mod: %s" % info['headers']['last-modified'])
566 output(u" MIME type: %s" % info['headers']['content-type'])
567 output(u" MD5 sum: %s" % info['headers']['etag'].strip('"'))
568 else:
569 info = s3.bucket_info(uri)
570 output(u"%s (bucket):" % uri.uri())
571 output(u" Location: %s" % info['bucket-location'])
572 acl = s3.get_acl(uri)
573 acl_grant_list = acl.getGrantList()
574 for grant in acl_grant_list:
575 output(u" ACL: %s: %s" % (grant['grantee'], grant['permission']))
576 if acl.isAnonRead():
577 output(u" URL: %s" % uri.public_url())
578 except S3Error, e:
579 if S3.codes.has_key(e.info["Code"]):
580 error(S3.codes[e.info["Code"]] % uri.bucket())
581 return
582 else:
583 raise
e5c6f6c @mludvig 2008-02-27 Michal Ludvig <michal@logix.cz>
mludvig authored
584
13fc0d5 @mludvig * s3cmd: Added support for remote-to-remote sync.
mludvig authored
585 def cmd_sync_remote2remote(args):
7ee7549 @mdomsch add more --delete-after to sync variations
authored
586 def _do_deletes(s3, dst_list):
587 # Delete items in destination that are not in source
588 if cfg.dry_run:
589 for key in dst_list:
590 output(u"delete: %s" % dst_list[key]['object_uri_str'])
591 else:
592 for key in dst_list:
593 uri = S3Uri(dst_list[key]['object_uri_str'])
594 s3.object_delete(uri)
595 output(u"deleted: '%s'" % uri)
596
d439efb @mludvig ATTENTION -- Mega WhiteSpace conversion !!!
mludvig authored
597 s3 = S3(Config())
598
599 # Normalise s3://uri (e.g. assert trailing slash)
600 destination_base = unicode(S3Uri(args[-1]))
601
602 src_list = fetch_remote_list(args[:-1], recursive = True, require_attribs = True)
603 dst_list = fetch_remote_list(destination_base, recursive = True, require_attribs = True)
604
605 src_count = len(src_list)
606 dst_count = len(dst_list)
607
608 info(u"Found %d source files, %d destination files" % (src_count, dst_count))
609
610 src_list, exclude_list = filter_exclude_include(src_list)
611
1703df7 @mdomsch Handle hardlinks and duplicate files
authored
612 src_list, dst_list, update_list, copy_pairs = compare_filelists(src_list, dst_list, src_remote = True, dst_remote = True, delay_updates = cfg.delay_updates)
d439efb @mludvig ATTENTION -- Mega WhiteSpace conversion !!!
mludvig authored
613
614 src_count = len(src_list)
c3deb6a @mdomsch add --delay-updates option
authored
615 update_count = len(update_list)
d439efb @mludvig ATTENTION -- Mega WhiteSpace conversion !!!
mludvig authored
616 dst_count = len(dst_list)
617
618 print(u"Summary: %d source files to copy, %d files at destination to delete" % (src_count, dst_count))
619
620 if src_count > 0:
621 ### Populate 'remote_uri' only if we've got something to sync from src to dst
622 for key in src_list:
623 src_list[key]['target_uri'] = destination_base + key
624
625 if cfg.dry_run:
626 for key in exclude_list:
627 output(u"exclude: %s" % unicodise(key))
628 if cfg.delete_removed:
629 for key in dst_list:
630 output(u"delete: %s" % dst_list[key]['object_uri_str'])
631 for key in src_list:
632 output(u"Sync: %s -> %s" % (src_list[key]['object_uri_str'], src_list[key]['target_uri']))
633 warning(u"Exitting now because of --dry-run")
634 return
635
1703df7 @mdomsch Handle hardlinks and duplicate files
authored
636 # if there are copy pairs, we can't do delete_before, on the chance
637 # we need one of the to-be-deleted files as a copy source.
638 if len(copy_pairs) > 0:
639 cfg.delete_after = True
640
d439efb @mludvig ATTENTION -- Mega WhiteSpace conversion !!!
mludvig authored
641 # Delete items in destination that are not in source
d5dc2c0 @mdomsch add --delete-after option for sync
authored
642 if cfg.delete_removed and not cfg.delete_after:
7ee7549 @mdomsch add more --delete-after to sync variations
authored
643 _do_deletes(s3, dst_list)
d439efb @mludvig ATTENTION -- Mega WhiteSpace conversion !!!
mludvig authored
644
c3deb6a @mdomsch add --delay-updates option
authored
645 def _upload(src_list, seq, src_count):
646 file_list = src_list.keys()
647 file_list.sort()
648 for file in file_list:
649 seq += 1
650 item = src_list[file]
651 src_uri = S3Uri(item['object_uri_str'])
652 dst_uri = S3Uri(item['target_uri'])
653 seq_label = "[%d of %d]" % (seq, src_count)
654 extra_headers = copy(cfg.extra_headers)
655 try:
656 response = s3.object_copy(src_uri, dst_uri, extra_headers)
657 output("File %(src)s copied to %(dst)s" % { "src" : src_uri, "dst" : dst_uri })
658 except S3Error, e:
659 error("File %(src)s could not be copied: %(e)s" % { "src" : src_uri, "e" : e })
660 return seq
661
d439efb @mludvig ATTENTION -- Mega WhiteSpace conversion !!!
mludvig authored
662 # Perform the synchronization of files
663 timestamp_start = time.time()
664 seq = 0
c3deb6a @mdomsch add --delay-updates option
authored
665 seq = _upload(src_list, seq, src_count + update_count)
666 seq = _upload(update_list, seq, src_count + update_count)
517ee93 @mdomsch remote_copy() doesn't need to know dst_list anymore
authored
667 n_copied, bytes_saved = remote_copy(s3, copy_pairs, destination_base)
c3deb6a @mdomsch add --delay-updates option
authored
668
d439efb @mludvig ATTENTION -- Mega WhiteSpace conversion !!!
mludvig authored
669 total_elapsed = time.time() - timestamp_start
670 outstr = "Done. Copied %d files in %0.1f seconds, %0.2f files/s" % (seq, total_elapsed, seq/total_elapsed)
671 if seq > 0:
672 output(outstr)
673 else:
674 info(outstr)
13fc0d5 @mludvig * s3cmd: Added support for remote-to-remote sync.
mludvig authored
675
d5dc2c0 @mdomsch add --delete-after option for sync
authored
676 # Delete items in destination that are not in source
677 if cfg.delete_removed and cfg.delete_after:
7ee7549 @mdomsch add more --delete-after to sync variations
authored
678 _do_deletes(s3, dst_list)
d5dc2c0 @mdomsch add --delete-after option for sync
authored
679
227fabf @mludvig * s3cmd: Migrated 'sync' remote->local to the new
mludvig authored
680 def cmd_sync_remote2local(args):
7ee7549 @mdomsch add more --delete-after to sync variations
authored
681 def _do_deletes(local_list):
682 for key in local_list:
683 os.unlink(local_list[key]['full_name'])
684 output(u"deleted: %s" % local_list[key]['full_name_unicode'])
685
d439efb @mludvig ATTENTION -- Mega WhiteSpace conversion !!!
mludvig authored
686 s3 = S3(Config())
687
688 destination_base = args[-1]
689 local_list, single_file_local = fetch_local_list(destination_base, recursive = True)
690 remote_list = fetch_remote_list(args[:-1], recursive = True, require_attribs = True)
691
692 local_count = len(local_list)
693 remote_count = len(remote_list)
694
695 info(u"Found %d remote files, %d local files" % (remote_count, local_count))
696
697 remote_list, exclude_list = filter_exclude_include(remote_list)
698
1703df7 @mdomsch Handle hardlinks and duplicate files
authored
699 remote_list, local_list, update_list, copy_pairs = compare_filelists(remote_list, local_list, src_remote = True, dst_remote = False, delay_updates = cfg.delay_updates)
d439efb @mludvig ATTENTION -- Mega WhiteSpace conversion !!!
mludvig authored
700
701 local_count = len(local_list)
702 remote_count = len(remote_list)
c3deb6a @mdomsch add --delay-updates option
authored
703 update_count = len(update_list)
ddb5ef9 @mdomsch handle remote->local transfers with local hardlink/copy if possible
authored
704 copy_pairs_count = len(copy_pairs)
d439efb @mludvig ATTENTION -- Mega WhiteSpace conversion !!!
mludvig authored
705
ddb5ef9 @mdomsch handle remote->local transfers with local hardlink/copy if possible
authored
706 info(u"Summary: %d remote files to download, %d local files to delete, %d local files to hardlink" % (remote_count + update_count, local_count, copy_pairs_count))
d439efb @mludvig ATTENTION -- Mega WhiteSpace conversion !!!
mludvig authored
707
c3deb6a @mdomsch add --delay-updates option
authored
708 def _set_local_filename(remote_list, destination_base):
709 if not os.path.isdir(destination_base):
710 ## We were either given a file name (existing or not) or want STDOUT
711 if len(remote_list) > 1:
712 raise ParameterError("Destination must be a directory when downloading multiple sources.")
713 remote_list[remote_list.keys()[0]]['local_filename'] = deunicodise(destination_base)
714 else:
715 if destination_base[-1] != os.path.sep:
716 destination_base += os.path.sep
717 for key in remote_list:
718 local_filename = destination_base + key
719 if os.path.sep != "/":
720 local_filename = os.path.sep.join(local_filename.split("/"))
721 remote_list[key]['local_filename'] = deunicodise(local_filename)
722
723 _set_local_filename(remote_list, destination_base)
724 _set_local_filename(update_list, destination_base)
725
d439efb @mludvig ATTENTION -- Mega WhiteSpace conversion !!!
mludvig authored
726 if cfg.dry_run:
727 for key in exclude_list:
728 output(u"exclude: %s" % unicodise(key))
729 if cfg.delete_removed:
730 for key in local_list:
731 output(u"delete: %s" % local_list[key]['full_name_unicode'])
732 for key in remote_list:
733 output(u"download: %s -> %s" % (remote_list[key]['object_uri_str'], remote_list[key]['local_filename']))
c3deb6a @mdomsch add --delay-updates option
authored
734 for key in update_list:
735 output(u"download: %s -> %s" % (update_list[key]['object_uri_str'], update_list[key]['local_filename']))
d439efb @mludvig ATTENTION -- Mega WhiteSpace conversion !!!
mludvig authored
736
737 warning(u"Exitting now because of --dry-run")
738 return
739
1703df7 @mdomsch Handle hardlinks and duplicate files
authored
740 # if there are copy pairs, we can't do delete_before, on the chance
741 # we need one of the to-be-deleted files as a copy source.
742 if len(copy_pairs) > 0:
743 cfg.delete_after = True
744
7ee7549 @mdomsch add more --delete-after to sync variations
authored
745 if cfg.delete_removed and not cfg.delete_after:
746 _do_deletes(local_list)
d439efb @mludvig ATTENTION -- Mega WhiteSpace conversion !!!
mludvig authored
747
c3deb6a @mdomsch add --delay-updates option
authored
748 def _download(remote_list, seq, total, total_size, dir_cache):
749 file_list = remote_list.keys()
750 file_list.sort()
751 for file in file_list:
752 seq += 1
753 item = remote_list[file]
754 uri = S3Uri(item['object_uri_str'])
755 dst_file = item['local_filename']
756 seq_label = "[%d of %d]" % (seq, total)
d439efb @mludvig ATTENTION -- Mega WhiteSpace conversion !!!
mludvig authored
757 try:
c3deb6a @mdomsch add --delay-updates option
authored
758 dst_dir = os.path.dirname(dst_file)
759 if not dir_cache.has_key(dst_dir):
760 dir_cache[dst_dir] = Utils.mkdir_with_parents(dst_dir)
761 if dir_cache[dst_dir] == False:
762 warning(u"%s: destination directory not writable: %s" % (file, dst_dir))
d439efb @mludvig ATTENTION -- Mega WhiteSpace conversion !!!
mludvig authored
763 continue
c3deb6a @mdomsch add --delay-updates option
authored
764 try:
765 open_flags = os.O_CREAT
766 open_flags |= os.O_TRUNC
767 # open_flags |= os.O_EXCL
768
769 debug(u"dst_file=%s" % unicodise(dst_file))
770 # This will have failed should the file exist
771 os.close(os.open(dst_file, open_flags))
772 # Yeah I know there is a race condition here. Sadly I don't know how to open() in exclusive mode.
773 dst_stream = open(dst_file, "wb")
774 response = s3.object_get(uri, dst_stream, extra_label = seq_label)
775 dst_stream.close()
776 if response['headers'].has_key('x-amz-meta-s3cmd-attrs') and cfg.preserve_attrs:
1703df7 @mdomsch Handle hardlinks and duplicate files
authored
777 attrs = parse_attrs_header(response['headers']['x-amz-meta-s3cmd-attrs'])
c3deb6a @mdomsch add --delay-updates option
authored
778 if attrs.has_key('mode'):
779 os.chmod(dst_file, int(attrs['mode']))
780 if attrs.has_key('mtime') or attrs.has_key('atime'):
781 mtime = attrs.has_key('mtime') and int(attrs['mtime']) or int(time.time())
782 atime = attrs.has_key('atime') and int(attrs['atime']) or int(time.time())
783 os.utime(dst_file, (atime, mtime))
784 ## FIXME: uid/gid / uname/gname handling comes here! TODO
785 except OSError, e:
786 try: dst_stream.close()
787 except: pass
788 if e.errno == errno.EEXIST:
789 warning(u"%s exists - not overwriting" % (dst_file))
790 continue
791 if e.errno in (errno.EPERM, errno.EACCES):
792 warning(u"%s not writable: %s" % (dst_file, e.strerror))
793 continue
794 if e.errno == errno.EISDIR:
795 warning(u"%s is a directory - skipping over" % dst_file)
796 continue
797 raise e
798 except KeyboardInterrupt:
799 try: dst_stream.close()
800 except: pass
801 warning(u"Exiting after keyboard interrupt")
802 return
803 except Exception, e:
804 try: dst_stream.close()
805 except: pass
806 error(u"%s: %s" % (file, e))
d439efb @mludvig ATTENTION -- Mega WhiteSpace conversion !!!
mludvig authored
807 continue
c3deb6a @mdomsch add --delay-updates option
authored
808 # We have to keep repeating this call because
809 # Python 2.4 doesn't support try/except/finally
810 # construction :-(
d439efb @mludvig ATTENTION -- Mega WhiteSpace conversion !!!
mludvig authored
811 try: dst_stream.close()
812 except: pass
c3deb6a @mdomsch add --delay-updates option
authored
813 except S3DownloadError, e:
814 error(u"%s: download failed too many times. Skipping that file." % file)
d439efb @mludvig ATTENTION -- Mega WhiteSpace conversion !!!
mludvig authored
815 continue
c3deb6a @mdomsch add --delay-updates option
authored
816 speed_fmt = formatSize(response["speed"], human_readable = True, floating_point = True)
817 if not Config().progress_meter:
818 output(u"File '%s' stored as '%s' (%d bytes in %0.1f seconds, %0.2f %sB/s) %s" %
819 (uri, unicodise(dst_file), response["size"], response["elapsed"], speed_fmt[0], speed_fmt[1],
820 seq_label))
821 total_size += response["size"]
822 return seq, total_size
d439efb @mludvig ATTENTION -- Mega WhiteSpace conversion !!!
mludvig authored
823
c3deb6a @mdomsch add --delay-updates option
authored
824 total_size = 0
825 total_elapsed = 0.0
826 timestamp_start = time.time()
827 dir_cache = {}
828 seq = 0
829 seq, total_size = _download(remote_list, seq, remote_count + update_count, total_size, dir_cache)
830 seq, total_size = _download(update_list, seq, remote_count + update_count, total_size, dir_cache)
ddb5ef9 @mdomsch handle remote->local transfers with local hardlink/copy if possible
authored
831 local_hardlink(copy_pairs, destination_base)
c3deb6a @mdomsch add --delay-updates option
authored
832
d439efb @mludvig ATTENTION -- Mega WhiteSpace conversion !!!
mludvig authored
833 total_elapsed = time.time() - timestamp_start
834 speed_fmt = formatSize(total_size/total_elapsed, human_readable = True, floating_point = True)
835
836 # Only print out the result if any work has been done or
837 # if the user asked for verbose output
838 outstr = "Done. Downloaded %d bytes in %0.1f seconds, %0.2f %sB/s" % (total_size, total_elapsed, speed_fmt[0], speed_fmt[1])
839 if total_size > 0:
840 output(outstr)
841 else:
842 info(outstr)
01fe3a2 @mludvig * s3cmd: Refactored cmd_sync() in preparation
mludvig authored
843
7ee7549 @mdomsch add more --delete-after to sync variations
authored
844 if cfg.delete_removed and cfg.delete_after:
845 _do_deletes(local_list)
846
ddb5ef9 @mdomsch handle remote->local transfers with local hardlink/copy if possible
authored
847 def local_hardlink(copy_pairs, destination_base):
848 for (dst1, dst2) in copy_pairs:
849 try:
850 os.link(destination_base + dst1, destination_base + dst2)
851 debug(u"Hardlinking %s to %s" % (destination_base + dst1, destination_base + dst2))
852 except:
853 shutil.copy2(destination_base + dst1, destination_base + dst2)
854 debug(u"Hardlinking unavailable, copying %s to %s" % (destination_base + dst1, destination_base + dst2))
855
517ee93 @mdomsch remote_copy() doesn't need to know dst_list anymore
authored
856 def remote_copy(s3, copy_pairs, destination_base):
1703df7 @mdomsch Handle hardlinks and duplicate files
authored
857 saved_bytes = 0
858 for (dst1, dst2) in copy_pairs:
859 debug(u"Remote Copying from %s to %s" % (dst1, dst2))
d4e5a52 @mdomsch hardlink/copy fix
authored
860 dst1_uri = S3Uri(destination_base + dst1)
861 dst2_uri = S3Uri(destination_base + dst2)
1703df7 @mdomsch Handle hardlinks and duplicate files
authored
862 extra_headers = copy(cfg.extra_headers)
863 try:
d4e5a52 @mdomsch hardlink/copy fix
authored
864 s3.object_copy(dst1_uri, dst2_uri, extra_headers)
865 info = s3.object_info(dst2_uri)
866 saved_bytes = saved_bytes + int(info['headers']['content-length'])
1703df7 @mdomsch Handle hardlinks and duplicate files
authored
867 except:
868 raise
869 return (len(copy_pairs), saved_bytes)
870
871
d7251cc @mludvig * s3cmd: Migrated 'sync' local->remote to the new
mludvig authored
872 def cmd_sync_local2remote(args):
1703df7 @mdomsch Handle hardlinks and duplicate files
authored
873 def _build_attr_header(local_list, src):
d439efb @mludvig ATTENTION -- Mega WhiteSpace conversion !!!
mludvig authored
874 import pwd, grp
875 attrs = {}
876 for attr in cfg.preserve_attrs_list:
877 if attr == 'uname':
878 try:
1703df7 @mdomsch Handle hardlinks and duplicate files
authored
879 val = pwd.getpwuid(local_list[src]['uid']).pw_name
d439efb @mludvig ATTENTION -- Mega WhiteSpace conversion !!!
mludvig authored
880 except KeyError:
881 attr = "uid"
2afb06b @mdomsch fix getting uid
authored
882 val = local_list[src].get('uid')
1703df7 @mdomsch Handle hardlinks and duplicate files
authored
883 warning(u"%s: Owner username not known. Storing UID=%d instead." % (src, val))
d439efb @mludvig ATTENTION -- Mega WhiteSpace conversion !!!
mludvig authored
884 elif attr == 'gname':
885 try:
1703df7 @mdomsch Handle hardlinks and duplicate files
authored
886 val = grp.getgrgid(local_list[src].get('gid')).gr_name
d439efb @mludvig ATTENTION -- Mega WhiteSpace conversion !!!
mludvig authored
887 except KeyError:
888 attr = "gid"
1703df7 @mdomsch Handle hardlinks and duplicate files
authored
889 val = local_list[src].get('gid')
890 warning(u"%s: Owner groupname not known. Storing GID=%d instead." % (src, val))
891 elif attr == 'md5':
892 val = local_list.get_md5(src)
d439efb @mludvig ATTENTION -- Mega WhiteSpace conversion !!!
mludvig authored
893 else:
1703df7 @mdomsch Handle hardlinks and duplicate files
authored
894 val = getattr(local_list[src]['sr'], 'st_' + attr)
d439efb @mludvig ATTENTION -- Mega WhiteSpace conversion !!!
mludvig authored
895 attrs[attr] = val
1703df7 @mdomsch Handle hardlinks and duplicate files
authored
896
897 if 'md5' in attrs and attrs['md5'] is None:
898 del attrs['md5']
899
d439efb @mludvig ATTENTION -- Mega WhiteSpace conversion !!!
mludvig authored
900 result = ""
901 for k in attrs: result += "%s:%s/" % (k, attrs[k])
902 return { 'x-amz-meta-s3cmd-attrs' : result[:-1] }
903
7ee7549 @mdomsch add more --delete-after to sync variations
authored
904 def _do_deletes(s3, remote_list):
905 for key in remote_list:
906 uri = S3Uri(remote_list[key]['object_uri_str'])
907 s3.object_delete(uri)
908 output(u"deleted: '%s'" % uri)
909
fb6441c @mdomsch sync: refactor parent/child and single process code
authored
910 def _single_process(local_list):
911 for dest in destinations:
912 ## Normalize URI to convert s3://bkt to s3://bkt/ (trailing slash)
913 destination_base_uri = S3Uri(dest)
914 if destination_base_uri.type != 's3':
915 raise ParameterError("Destination must be S3Uri. Got: %s" % destination_base_uri)
916 destination_base = str(destination_base_uri)
917 _child(destination_base, local_list)
918
919 def _parent():
920 # Now that we've done all the disk I/O to look at the local file system and
921 # calculate the md5 for each file, fork for each destination to upload to them separately
922 # and in parallel
923 child_pids = []
924
925 for dest in destinations:
926 ## Normalize URI to convert s3://bkt to s3://bkt/ (trailing slash)
927 destination_base_uri = S3Uri(dest)
928 if destination_base_uri.type != 's3':
929 raise ParameterError("Destination must be S3Uri. Got: %s" % destination_base_uri)
930 destination_base = str(destination_base_uri)
931 child_pid = os.fork()
932 if child_pid == 0:
933 _child(destination_base, local_list)
934 os._exit(0)
935 else:
936 child_pids.append(child_pid)
07c9e2d @mdomsch sync: add --add-destination, parallelize uploads to multiple destination...
authored
937
938 while len(child_pids):
939 (pid, status) = os.wait()
940 child_pids.remove(pid)
941 return
942
fb6441c @mdomsch sync: refactor parent/child and single process code
authored
943 def _child(destination_base, local_list):
944 def _set_remote_uri(local_list, destination_base, single_file_local):
945 if len(local_list) > 0:
946 ## Populate 'remote_uri' only if we've got something to upload
947 if not destination_base.endswith("/"):
948 if not single_file_local:
949 raise ParameterError("Destination S3 URI must end with '/' (ie must refer to a directory on the remote side).")
950 local_list[local_list.keys()[0]]['remote_uri'] = unicodise(destination_base)
951 else:
952 for key in local_list:
953 local_list[key]['remote_uri'] = unicodise(destination_base + key)
954
955 def _upload(local_list, seq, total, total_size):
956 file_list = local_list.keys()
957 file_list.sort()
958 for file in file_list:
959 seq += 1
960 item = local_list[file]
961 src = item['full_name']
962 uri = S3Uri(item['remote_uri'])
963 seq_label = "[%d of %d]" % (seq, total)
964 extra_headers = copy(cfg.extra_headers)
965 try:
966 if cfg.preserve_attrs:
967 attr_header = _build_attr_header(local_list, file)
968 debug(u"attr_header: %s" % attr_header)
969 extra_headers.update(attr_header)
970 response = s3.object_put(src, uri, extra_headers, extra_label = seq_label)
971 except InvalidFileError, e:
972 warning(u"File can not be uploaded: %s" % e)
973 continue
974 except S3UploadError, e:
975 error(u"%s: upload failed too many times. Skipping that file." % item['full_name_unicode'])
976 continue
977 speed_fmt = formatSize(response["speed"], human_readable = True, floating_point = True)
978 if not cfg.progress_meter:
979 output(u"File '%s' stored as '%s' (%d bytes in %0.1f seconds, %0.2f %sB/s) %s" %
980 (item['full_name_unicode'], uri, response["size"], response["elapsed"],
981 speed_fmt[0], speed_fmt[1], seq_label))
982 total_size += response["size"]
983 uploaded_objects_list.append(uri.object())
984 return seq, total_size
d439efb @mludvig ATTENTION -- Mega WhiteSpace conversion !!!
mludvig authored
985
fb6441c @mdomsch sync: refactor parent/child and single process code
authored
986 remote_list = fetch_remote_list(destination_base, recursive = True, require_attribs = True)
d439efb @mludvig ATTENTION -- Mega WhiteSpace conversion !!!
mludvig authored
987
fb6441c @mdomsch sync: refactor parent/child and single process code
authored
988 local_count = len(local_list)
989 remote_count = len(remote_list)
d439efb @mludvig ATTENTION -- Mega WhiteSpace conversion !!!
mludvig authored
990
fb6441c @mdomsch sync: refactor parent/child and single process code
authored
991 info(u"Found %d local files, %d remote files" % (local_count, remote_count))
d439efb @mludvig ATTENTION -- Mega WhiteSpace conversion !!!
mludvig authored
992
fb6441c @mdomsch sync: refactor parent/child and single process code
authored
993 local_list, exclude_list = filter_exclude_include(local_list)
c3deb6a @mdomsch add --delay-updates option
authored
994
fb6441c @mdomsch sync: refactor parent/child and single process code
authored
995 if single_file_local and len(local_list) == 1 and len(remote_list) == 1:
996 ## Make remote_key same as local_key for comparison if we're dealing with only one file
997 remote_list_entry = remote_list[remote_list.keys()[0]]
998 # Flush remote_list, by the way
999 remote_list = { local_list.keys()[0] : remote_list_entry }
d439efb @mludvig ATTENTION -- Mega WhiteSpace conversion !!!
mludvig authored
1000
fb6441c @mdomsch sync: refactor parent/child and single process code
authored
1001 local_list, remote_list, update_list, copy_pairs = compare_filelists(local_list, remote_list, src_remote = False, dst_remote = True, delay_updates = cfg.delay_updates)
d439efb @mludvig ATTENTION -- Mega WhiteSpace conversion !!!
mludvig authored
1002
fb6441c @mdomsch sync: refactor parent/child and single process code
authored
1003 local_count = len(local_list)
1004 update_count = len(update_list)
1005 copy_count = len(copy_pairs)
1006 remote_count = len(remote_list)
d439efb @mludvig ATTENTION -- Mega WhiteSpace conversion !!!
mludvig authored
1007
fb6441c @mdomsch sync: refactor parent/child and single process code
authored
1008 info(u"Summary: %d local files to upload, %d files to remote copy, %d remote files to delete" % (local_count + update_count, copy_count, remote_count))
c3deb6a @mdomsch add --delay-updates option
authored
1009
fb6441c @mdomsch sync: refactor parent/child and single process code
authored
1010 _set_remote_uri(local_list, destination_base, single_file_local)
1011 _set_remote_uri(update_list, destination_base, single_file_local)
d439efb @mludvig ATTENTION -- Mega WhiteSpace conversion !!!
mludvig authored
1012
fb6441c @mdomsch sync: refactor parent/child and single process code
authored
1013 if cfg.dry_run:
1014 for key in exclude_list:
1015 output(u"exclude: %s" % unicodise(key))
1016 for key in local_list:
1017 output(u"upload: %s -> %s" % (local_list[key]['full_name_unicode'], local_list[key]['remote_uri']))
1018 for key in update_list:
1019 output(u"upload: %s -> %s" % (update_list[key]['full_name_unicode'], update_list[key]['remote_uri']))
1020 for (dst1, dst2) in copy_pairs:
1021 output(u"remote copy: %s -> %s" % (dst1['object_key'], remote_list[dst2]['object_key']))
1022 if cfg.delete_removed:
1023 for key in remote_list:
1024 output(u"delete: %s" % remote_list[key]['object_uri_str'])
1025
1026 warning(u"Exitting now because of --dry-run")
1027 return
d439efb @mludvig ATTENTION -- Mega WhiteSpace conversion !!!
mludvig authored
1028
fb6441c @mdomsch sync: refactor parent/child and single process code
authored
1029 # if there are copy pairs, we can't do delete_before, on the chance
1030 # we need one of the to-be-deleted files as a copy source.
1031 if len(copy_pairs) > 0:
1032 cfg.delete_after = True
1033
1034 if cfg.delete_removed and not cfg.delete_after:
1035 _do_deletes(s3, remote_list)
1036
1037 uploaded_objects_list = []
1038 total_size = 0
1039 total_elapsed = 0.0
1040 timestamp_start = time.time()
1041 n, total_size = _upload(local_list, 0, local_count, total_size)
1042 n, total_size = _upload(update_list, n, local_count, total_size)
1043 n_copies, saved_bytes = remote_copy(s3, copy_pairs, destination_base)
1044 if cfg.delete_removed and cfg.delete_after:
1045 _do_deletes(s3, remote_list)
1046 total_elapsed = time.time() - timestamp_start
1047 total_speed = total_elapsed and total_size/total_elapsed or 0.0
1048 speed_fmt = formatSize(total_speed, human_readable = True, floating_point = True)
1049
1050 # Only print out the result if any work has been done or
1051 # if the user asked for verbose output
1052 outstr = "Done. Uploaded %d bytes in %0.1f seconds, %0.2f %sB/s. Copied %d files saving %d bytes transfer." % (total_size, total_elapsed, speed_fmt[0], speed_fmt[1], n_copies, saved_bytes)
1053 if total_size + saved_bytes > 0:
1054 output(outstr)
1055 else:
1056 info(outstr)
1703df7 @mdomsch Handle hardlinks and duplicate files
authored
1057
fb6441c @mdomsch sync: refactor parent/child and single process code
authored
1058 if cfg.invalidate_on_cf:
1059 if len(uploaded_objects_list) == 0:
1060 info("Nothing to invalidate in CloudFront")
1061 else:
1062 # 'uri' from the last iteration is still valid at this point
1063 cf = CloudFront(cfg)
1064 result = cf.InvalidateObjects(uri, uploaded_objects_list)
1065 if result['status'] == 201:
1066 output("Created invalidation request for %d paths" % len(uploaded_objects_list))
1067 output("Check progress with: s3cmd cfinvalinfo cf://%s/%s" % (result['dist_id'], result['request_id']))
d439efb @mludvig ATTENTION -- Mega WhiteSpace conversion !!!
mludvig authored
1068
fb6441c @mdomsch sync: refactor parent/child and single process code
authored
1069 return
d439efb @mludvig ATTENTION -- Mega WhiteSpace conversion !!!
mludvig authored
1070
fb6441c @mdomsch sync: refactor parent/child and single process code
authored
1071 # main execution
1072 s3 = S3(cfg)
c3deb6a @mdomsch add --delay-updates option
authored
1073
fb6441c @mdomsch sync: refactor parent/child and single process code
authored
1074 if cfg.encrypt:
1075 error(u"S3cmd 'sync' doesn't yet support GPG encryption, sorry.")
1076 error(u"Either use unconditional 's3cmd put --recursive'")
1077 error(u"or disable encryption with --no-encrypt parameter.")
1078 sys.exit(1)
1703df7 @mdomsch Handle hardlinks and duplicate files
authored
1079
fb6441c @mdomsch sync: refactor parent/child and single process code
authored
1080 local_list, single_file_local = fetch_local_list(args[:-1], recursive = True)
d439efb @mludvig ATTENTION -- Mega WhiteSpace conversion !!!
mludvig authored
1081
fb6441c @mdomsch sync: refactor parent/child and single process code
authored
1082 destinations = [args[-1]]
1083 if cfg.additional_destinations:
1084 destinations = destinations + cfg.additional_destinations
1085
1086 if 'fork' not in os.__all__ or len(destinations) < 2:
1087 _single_process(local_list)
d439efb @mludvig ATTENTION -- Mega WhiteSpace conversion !!!
mludvig authored
1088 else:
fb6441c @mdomsch sync: refactor parent/child and single process code
authored
1089 _parent()
d439efb @mludvig ATTENTION -- Mega WhiteSpace conversion !!!
mludvig authored
1090
07c9e2d @mdomsch sync: add --add-destination, parallelize uploads to multiple destination...
authored
1091
01fe3a2 @mludvig * s3cmd: Refactored cmd_sync() in preparation
mludvig authored
1092 def cmd_sync(args):
d439efb @mludvig ATTENTION -- Mega WhiteSpace conversion !!!
mludvig authored
1093 if (len(args) < 2):
1094 raise ParameterError("Too few parameters! Expected: %s" % commands['sync']['param'])
01fe3a2 @mludvig * s3cmd: Refactored cmd_sync() in preparation
mludvig authored
1095
d439efb @mludvig ATTENTION -- Mega WhiteSpace conversion !!!
mludvig authored
1096 if S3Uri(args[0]).type == "file" and S3Uri(args[-1]).type == "s3":
1097 return cmd_sync_local2remote(args)
1098 if S3Uri(args[0]).type == "s3" and S3Uri(args[-1]).type == "file":
1099 return cmd_sync_remote2local(args)
1100 if S3Uri(args[0]).type == "s3" and S3Uri(args[-1]).type == "s3":
1101 return cmd_sync_remote2remote(args)
1102 raise ParameterError("Invalid source/destination: '%s'" % "' '".join(args))
585c735 @mludvig * s3cmd: New command 'setacl'.
mludvig authored
1103
1104 def cmd_setacl(args):
d439efb @mludvig ATTENTION -- Mega WhiteSpace conversion !!!
mludvig authored
1105 def _update_acl(uri, seq_label = ""):
1106 something_changed = False
1107 acl = s3.get_acl(uri)
1108 debug(u"acl: %s - %r" % (uri, acl.grantees))
1109 if cfg.acl_public == True:
1110 if acl.isAnonRead():
1111 info(u"%s: already Public, skipping %s" % (uri, seq_label))
1112 else:
1113 acl.grantAnonRead()
1114 something_changed = True
1115 elif cfg.acl_public == False: # we explicitely check for False, because it could be None
1116 if not acl.isAnonRead():
1117 info(u"%s: already Private, skipping %s" % (uri, seq_label))
1118 else:
1119 acl.revokeAnonRead()
1120 something_changed = True
1121
1122 # update acl with arguments
1123 # grant first and revoke later, because revoke has priority
1124 if cfg.acl_grants:
1125 something_changed = True
1126 for grant in cfg.acl_grants:
1127 acl.grant(**grant);
1128
1129 if cfg.acl_revokes:
1130 something_changed = True
1131 for revoke in cfg.acl_revokes:
1132 acl.revoke(**revoke);
1133
1134 if not something_changed:
1135 return
1136
1137 retsponse = s3.set_acl(uri, acl)
1138 if retsponse['status'] == 200:
1139 if cfg.acl_public in (True, False):
1140 output(u"%s: ACL set to %s %s" % (uri, set_to_acl, seq_label))
1141 else:
1142 output(u"%s: ACL updated" % uri)
1143
1144 s3 = S3(cfg)
1145
1146 set_to_acl = cfg.acl_public and "Public" or "Private"
1147
1148 if not cfg.recursive:
1149 old_args = args
1150 args = []
1151 for arg in old_args:
1152 uri = S3Uri(arg)
1153 if not uri.has_object():
1154 if cfg.acl_public != None:
1155 info("Setting bucket-level ACL for %s to %s" % (uri.uri(), set_to_acl))
1156 else:
1157 info("Setting bucket-level ACL for %s" % (uri.uri()))
1158 if not cfg.dry_run:
1159 _update_acl(uri)
1160 else:
1161 args.append(arg)
1162
1163 remote_list = fetch_remote_list(args)
1164 remote_list, exclude_list = filter_exclude_include(remote_list)
1165
1166 remote_count = len(remote_list)
1167
1168 info(u"Summary: %d remote files to update" % remote_count)
1169
1170 if cfg.dry_run:
1171 for key in exclude_list:
1172 output(u"exclude: %s" % unicodise(key))
1173 for key in remote_list:
1174 output(u"setacl: %s" % remote_list[key]['object_uri_str'])
1175
1176 warning(u"Exitting now because of --dry-run")
1177 return
1178
1179 seq = 0
1180 for key in remote_list:
1181 seq += 1
1182 seq_label = "[%d of %d]" % (seq, remote_count)
1183 uri = S3Uri(remote_list[key]['object_uri_str'])
1184 _update_acl(uri, seq_label)
585c735 @mludvig * s3cmd: New command 'setacl'.
mludvig authored
1185
cb0bbae @mludvig * s3cmd, S3/AccessLog.py, ...: Added [accesslog] command.
mludvig authored
1186 def cmd_accesslog(args):
d439efb @mludvig ATTENTION -- Mega WhiteSpace conversion !!!
mludvig authored
1187 s3 = S3(cfg)
1188 bucket_uri = S3Uri(args.pop())
1189 if bucket_uri.object():
1190 raise ParameterError("Only bucket name is required for [accesslog] command")
1191 if cfg.log_target_prefix == False:
1192 accesslog, response = s3.set_accesslog(bucket_uri, enable = False)
1193 elif cfg.log_target_prefix:
1194 log_target_prefix_uri = S3Uri(cfg.log_target_prefix)
1195 if log_target_prefix_uri.type != "s3":
1196 raise ParameterError("--log-target-prefix must be a S3 URI")
1197 accesslog, response = s3.set_accesslog(bucket_uri, enable = True, log_target_prefix_uri = log_target_prefix_uri, acl_public = cfg.acl_public)
1198 else: # cfg.log_target_prefix == None
1199 accesslog = s3.get_accesslog(bucket_uri)
1200
1201 output(u"Access logging for: %s" % bucket_uri.uri())
1202 output(u" Logging Enabled: %s" % accesslog.isLoggingEnabled())
1203 if accesslog.isLoggingEnabled():
1204 output(u" Target prefix: %s" % accesslog.targetPrefix().uri())
1205 #output(u" Public Access: %s" % accesslog.isAclPublic())
1206
0b8ea55 @mludvig * s3cmd: New command [sign]
mludvig authored
1207 def cmd_sign(args):
d439efb @mludvig ATTENTION -- Mega WhiteSpace conversion !!!
mludvig authored
1208 string_to_sign = args.pop()
1209 debug("string-to-sign: %r" % string_to_sign)
1210 signature = Utils.sign_string(string_to_sign)
1211 output("Signature: %s" % signature)
0b8ea55 @mludvig * s3cmd: New command [sign]
mludvig authored
1212
3c07424 @mludvig * s3cmd: New [fixbucket] command for fixing invalid object
mludvig authored
1213 def cmd_fixbucket(args):
d439efb @mludvig ATTENTION -- Mega WhiteSpace conversion !!!
mludvig authored
1214 def _unescape(text):
1215 ##
1216 # Removes HTML or XML character references and entities from a text string.
1217 #
1218 # @param text The HTML (or XML) source text.
1219 # @return The plain text, as a Unicode string, if necessary.
1220 #
1221 # From: http://effbot.org/zone/re-sub.htm#unescape-html
1222 def _unescape_fixup(m):
1223 text = m.group(0)
1224 if not htmlentitydefs.name2codepoint.has_key('apos'):
1225 htmlentitydefs.name2codepoint['apos'] = ord("'")
1226 if text[:2] == "&#":
1227 # character reference
1228 try:
1229 if text[:3] == "&#x":
1230 return unichr(int(text[3:-1], 16))
1231 else:
1232 return unichr(int(text[2:-1]))
1233 except ValueError:
1234 pass
1235 else:
1236 # named entity
1237 try:
1238 text = unichr(htmlentitydefs.name2codepoint[text[1:-1]])
1239 except KeyError:
1240 pass
1241 return text # leave as is
1242 text = text.encode('ascii', 'xmlcharrefreplace')
1243 return re.sub("&#?\w+;", _unescape_fixup, text)
1244
1245 cfg.urlencoding_mode = "fixbucket"
1246 s3 = S3(cfg)
1247
1248 count = 0
1249 for arg in args:
1250 culprit = S3Uri(arg)
1251 if culprit.type != "s3":
1252 raise ParameterError("Expecting S3Uri instead of: %s" % arg)
1253 response = s3.bucket_list_noparse(culprit.bucket(), culprit.object(), recursive = True)
1254 r_xent = re.compile("&#x[\da-fA-F]+;")
1255 response['data'] = unicode(response['data'], 'UTF-8')
1256 keys = re.findall("<Key>(.*?)</Key>", response['data'], re.MULTILINE)
1257 debug("Keys: %r" % keys)
1258 for key in keys:
1259 if r_xent.search(key):
1260 info("Fixing: %s" % key)
1261 debug("Step 1: Transforming %s" % key)
1262 key_bin = _unescape(key)
1263 debug("Step 2: ... to %s" % key_bin)
1264 key_new = replace_nonprintables(key_bin)
1265 debug("Step 3: ... then to %s" % key_new)
1266 src = S3Uri("s3://%s/%s" % (culprit.bucket(), key_bin))
1267 dst = S3Uri("s3://%s/%s" % (culprit.bucket(), key_new))
1268 resp_move = s3.object_move(src, dst)
1269 if resp_move['status'] == 200:
1270 output("File %r renamed to %s" % (key_bin, key_new))
1271 count += 1
1272 else:
1273 error("Something went wrong for: %r" % key)
1274 error("Please report the problem to s3tools-bugs@lists.sourceforge.net")
1275 if count > 0:
1276 warning("Fixed %d files' names. Their ACL were reset to Private." % count)
1277 warning("Use 's3cmd setacl --acl-public s3://...' to make")
1278 warning("them publicly readable if required.")
3c07424 @mludvig * s3cmd: New [fixbucket] command for fixing invalid object
mludvig authored
1279
8ec1807 @mludvig 2007-05-27 Michal Ludvig <michal@logix.cz>
mludvig authored
1280 def resolve_list(lst, args):
d439efb @mludvig ATTENTION -- Mega WhiteSpace conversion !!!
mludvig authored
1281 retval = []
1282 for item in lst:
1283 retval.append(item % args)
1284 return retval
8ec1807 @mludvig 2007-05-27 Michal Ludvig <michal@logix.cz>
mludvig authored
1285
1286 def gpg_command(command, passphrase = ""):
d439efb @mludvig ATTENTION -- Mega WhiteSpace conversion !!!
mludvig authored
1287 debug("GPG command: " + " ".join(command))
1288 p = subprocess.Popen(command, stdin = subprocess.PIPE, stdout = subprocess.PIPE, stderr = subprocess.STDOUT)
1289 p_stdout, p_stderr = p.communicate(passphrase + "\n")
1290 debug("GPG output:")
1291 for line in p_stdout.split("\n"):
1292 debug("GPG: " + line)
1293 p_exitcode = p.wait()
1294 return p_exitcode
8ec1807 @mludvig 2007-05-27 Michal Ludvig <michal@logix.cz>
mludvig authored
1295
1296 def gpg_encrypt(filename):
d439efb @mludvig ATTENTION -- Mega WhiteSpace conversion !!!
mludvig authored
1297 tmp_filename = Utils.mktmpfile()
1298 args = {
1299 "gpg_command" : cfg.gpg_command,
1300 "passphrase_fd" : "0",
1301 "input_file" : filename,
1302 "output_file" : tmp_filename,
1303 }
1304 info(u"Encrypting file %(input_file)s to %(output_file)s..." % args)
1305 command = resolve_list(cfg.gpg_encrypt.split(" "), args)
1306 code = gpg_command(command, cfg.gpg_passphrase)
1307 return (code, tmp_filename, "gpg")
8ec1807 @mludvig 2007-05-27 Michal Ludvig <michal@logix.cz>
mludvig authored
1308
49731b4 @mludvig 2007-06-17 Michal Ludvig <michal@logix.cz>
mludvig authored
1309 def gpg_decrypt(filename, gpgenc_header = "", in_place = True):
d439efb @mludvig ATTENTION -- Mega WhiteSpace conversion !!!
mludvig authored
1310 tmp_filename = Utils.mktmpfile(filename)
1311 args = {
1312 "gpg_command" : cfg.gpg_command,
1313 "passphrase_fd" : "0",
1314 "input_file" : filename,
1315 "output_file" : tmp_filename,
1316 }
1317 info(u"Decrypting file %(input_file)s to %(output_file)s..." % args)
1318 command = resolve_list(cfg.gpg_decrypt.split(" "), args)
1319 code = gpg_command(command, cfg.gpg_passphrase)
1320 if code == 0 and in_place:
1321 debug(u"Renaming %s to %s" % (tmp_filename, filename))
1322 os.unlink(filename)
1323 os.rename(tmp_filename, filename)
1324 tmp_filename = filename
1325 return (code, tmp_filename)
8ec1807 @mludvig 2007-05-27 Michal Ludvig <michal@logix.cz>
mludvig authored
1326
a2340ee @mdrcode --configure now supports per-bucket checks
mdrcode authored
1327 def run_configure(config_file, args):
d439efb @mludvig ATTENTION -- Mega WhiteSpace conversion !!!
mludvig authored
1328 cfg = Config()
1329 options = [
1330 ("access_key", "Access Key", "Access key and Secret key are your identifiers for Amazon S3"),
1331 ("secret_key", "Secret Key"),
1332 ("gpg_passphrase", "Encryption password", "Encryption password is used to protect your files from reading\nby unauthorized persons while in transfer to S3"),
1333 ("gpg_command", "Path to GPG program"),
1334 ("use_https", "Use HTTPS protocol", "When using secure HTTPS protocol all communication with Amazon S3\nservers is protected from 3rd party eavesdropping. This method is\nslower than plain HTTP and can't be used if you're behind a proxy"),
1335 ("proxy_host", "HTTP Proxy server name", "On some networks all internet access must go through a HTTP proxy.\nTry setting it here if you can't conect to S3 directly"),
1336 ("proxy_port", "HTTP Proxy server port"),
1337 ]
1338 ## Option-specfic defaults
1339 if getattr(cfg, "gpg_command") == "":
1340 setattr(cfg, "gpg_command", find_executable("gpg"))
1341
1342 if getattr(cfg, "proxy_host") == "" and os.getenv("http_proxy"):
1343 re_match=re.match("(http://)?([^:]+):(\d+)", os.getenv("http_proxy"))
1344 if re_match:
1345 setattr(cfg, "proxy_host", re_match.groups()[1])
1346 setattr(cfg, "proxy_port", re_match.groups()[2])
1347
1348 try:
1349 while 1:
1350 output(u"\nEnter new values or accept defaults in brackets with Enter.")
1351 output(u"Refer to user manual for detailed description of all options.")
1352 for option in options:
1353 prompt = option[1]
1354 ## Option-specific handling
1355 if option[0] == 'proxy_host' and getattr(cfg, 'use_https') == True:
1356 setattr(cfg, option[0], "")
1357 continue
1358 if option[0] == 'proxy_port' and getattr(cfg, 'proxy_host') == "":
1359 setattr(cfg, option[0], 0)
1360 continue
1361
1362 try:
1363 val = getattr(cfg, option[0])
1364 if type(val) is bool:
1365 val = val and "Yes" or "No"
1366 if val not in (None, ""):
1367 prompt += " [%s]" % val
1368 except AttributeError:
1369 pass
1370
1371 if len(option) >= 3:
1372 output(u"\n%s" % option[2])
1373
1374 val = raw_input(prompt + ": ")
1375 if val != "":
1376 if type(getattr(cfg, option[0])) is bool:
1377 # Turn 'Yes' into True, everything else into False
1378 val = val.lower().startswith('y')
1379 setattr(cfg, option[0], val)
1380 output(u"\nNew settings:")
1381 for option in options:
1382 output(u" %s: %s" % (option[1], getattr(cfg, option[0])))
1383 val = raw_input("\nTest access with supplied credentials? [Y/n] ")
1384 if val.lower().startswith("y") or val == "":
1385 try:
a2340ee @mdrcode --configure now supports per-bucket checks
mdrcode authored
1386 # Default, we try to list 'all' buckets which requires
1387 # ListAllMyBuckets permission
1388 if len(args) == 0:
1389 output(u"Please wait, attempting to list all buckets...")
1390 S3(Config()).bucket_list("", "")
1391 else:
1392 # If user specified a bucket name directly, we check it and only it.
1393 # Thus, access check can succeed even if user only has access to
1394 # to a single bucket and not ListAllMyBuckets permission.
1395 output(u"Please wait, attempting to list bucket: " + args[0])
1396 uri = S3Uri(args[0])
1397 if uri.type == "s3" and uri.has_bucket():
1398 S3(Config()).bucket_list(uri.bucket(), "")
1399 else:
1400 raise Exception(u"Invalid bucket uri: " + args[0])
1401
d439efb @mludvig ATTENTION -- Mega WhiteSpace conversion !!!
mludvig authored
1402 output(u"Success. Your access key and secret key worked fine :-)")
1403
1404 output(u"\nNow verifying that encryption works...")
1405 if not getattr(cfg, "gpg_command") or not getattr(cfg, "gpg_passphrase"):
1406 output(u"Not configured. Never mind.")
1407 else:
1408 if not getattr(cfg, "gpg_command"):
1409 raise Exception("Path to GPG program not set")
1410 if not os.path.isfile(getattr(cfg, "gpg_command")):
1411 raise Exception("GPG program not found")
1412 filename = Utils.mktmpfile()
1413 f = open(filename, "w")
1414 f.write(os.sys.copyright)
1415 f.close()
1416 ret_enc = gpg_encrypt(filename)
1417 ret_dec = gpg_decrypt(ret_enc[1], ret_enc[2], False)
1418 hash = [
1419 Utils.hash_file_md5(filename),
1420 Utils.hash_file_md5(ret_enc[1]),
1421 Utils.hash_file_md5(ret_dec[1]),
1422 ]
1423 os.unlink(filename)
1424 os.unlink(ret_enc[1])
1425 os.unlink(ret_dec[1])
1426 if hash[0] == hash[2] and hash[0] != hash[1]:
1427 output ("Success. Encryption and decryption worked fine :-)")
1428 else:
1429 raise Exception("Encryption verification error.")
1430
1431 except Exception, e:
1432 error(u"Test failed: %s" % (e))
1433 val = raw_input("\nRetry configuration? [Y/n] ")
1434 if val.lower().startswith("y") or val == "":
1435 continue
1436
1437
1438 val = raw_input("\nSave settings? [y/N] ")
1439 if val.lower().startswith("y"):
1440 break
1441 val = raw_input("Retry configuration? [Y/n] ")
1442 if val.lower().startswith("n"):
1443 raise EOFError()
1444
1445 ## Overwrite existing config file, make it user-readable only
1446 old_mask = os.umask(0077)
1447 try:
1448 os.remove(config_file)
1449 except OSError, e:
1450 if e.errno != errno.ENOENT:
1451 raise
1452 f = open(config_file, "w")
1453 os.umask(old_mask)
1454 cfg.dump_config(f)
1455 f.close()
1456 output(u"Configuration saved to '%s'" % config_file)
1457
1458 except (EOFError, KeyboardInterrupt):
1459 output(u"\nConfiguration aborted. Changes were NOT saved.")
1460 return
1461
1462 except IOError, e:
1463 error(u"Writing config file failed: %s: %s" % (config_file, e.strerror))
1464 sys.exit(1)
5a736f0 @mludvig - Added interactive configurator (--configure)
mludvig authored
1465
7484d6c * s3cmd: Implemented --include and friends.
W. Tell authored
1466 def process_patterns_from_file(fname, patterns_list):
d439efb @mludvig ATTENTION -- Mega WhiteSpace conversion !!!
mludvig authored
1467 try:
1468 fn = open(fname, "rt")
1469 except IOError, e:
1470 error(e)
1471 sys.exit(1)
1472 for pattern in fn:
1473 pattern = pattern.strip()
1474 if re.match("^#", pattern) or re.match("^\s*$", pattern):
1475 continue
1476 debug(u"%s: adding rule: %s" % (fname, pattern))
1477 patterns_list.append(pattern)
1478
1479 return patterns_list
7484d6c * s3cmd: Implemented --include and friends.
W. Tell authored
1480
1481 def process_patterns(patterns_list, patterns_from, is_glob, option_txt = ""):
d439efb @mludvig ATTENTION -- Mega WhiteSpace conversion !!!
mludvig authored
1482 """
1483 process_patterns(patterns, patterns_from, is_glob, option_txt = "")
1484 Process --exclude / --include GLOB and REGEXP patterns.
1485 'option_txt' is 'exclude' / 'include' / 'rexclude' / 'rinclude'
1486 Returns: patterns_compiled, patterns_text
1487 """
1488
1489 patterns_compiled = []
1490 patterns_textual = {}
1491
1492 if patterns_list is None:
1493 patterns_list = []
1494
1495 if patterns_from:
1496 ## Append patterns from glob_from
1497 for fname in patterns_from:
1498 debug(u"processing --%s-from %s" % (option_txt, fname))
1499 patterns_list = process_patterns_from_file(fname, patterns_list)
1500
1501 for pattern in patterns_list:
1502 debug(u"processing %s rule: %s" % (option_txt, patterns_list))
1503 if is_glob:
1504 pattern = glob.fnmatch.translate(pattern)
1505 r = re.compile(pattern)
1506 patterns_compiled.append(r)
1507 patterns_textual[r] = pattern
1508
1509 return patterns_compiled, patterns_textual
2d7d554 @mludvig * s3cmd, s3cmd.1: Added GLOB (shell-style wildcard) exclude, renamed
mludvig authored
1510
b3488ba @mludvig * S3/CloudFront.py: New module for CloudFront support.
mludvig authored
1511 def get_commands_list():
d439efb @mludvig ATTENTION -- Mega WhiteSpace conversion !!!
mludvig authored
1512 return [
1513 {"cmd":"mb", "label":"Make bucket", "param":"s3://BUCKET", "func":cmd_bucket_create, "argc":1},
1514 {"cmd":"rb", "label":"Remove bucket", "param":"s3://BUCKET", "func":cmd_bucket_delete, "argc":1},
1515 {"cmd":"ls", "label":"List objects or buckets", "param":"[s3://BUCKET[/PREFIX]]", "func":cmd_ls, "argc":0},
1516 {"cmd":"la", "label":"List all object in all buckets", "param":"", "func":cmd_buckets_list_all_all, "argc":0},
1517 {"cmd":"put", "label":"Put file into bucket", "param":"FILE [FILE...] s3://BUCKET[/PREFIX]", "func":cmd_object_put, "argc":2},
1518 {"cmd":"get", "label":"Get file from bucket", "param":"s3://BUCKET/OBJECT LOCAL_FILE", "func":cmd_object_get, "argc":1},
1519 {"cmd":"del", "label":"Delete file from bucket", "param":"s3://BUCKET/OBJECT", "func":cmd_object_del, "argc":1},
1520 #{"cmd":"mkdir", "label":"Make a virtual S3 directory", "param":"s3://BUCKET/path/to/dir", "func":cmd_mkdir, "argc":1},
1521 {"cmd":"sync", "label":"Synchronize a directory tree to S3", "param":"LOCAL_DIR s3://BUCKET[/PREFIX] or s3://BUCKET[/PREFIX] LOCAL_DIR", "func":cmd_sync, "argc":2},
1522 {"cmd":"du", "label":"Disk usage by buckets", "param":"[s3://BUCKET[/PREFIX]]", "func":cmd_du, "argc":0},
1523 {"cmd":"info", "label":"Get various information about Buckets or Files", "param":"s3://BUCKET[/OBJECT]", "func":cmd_info, "argc":1},
1524 {"cmd":"cp", "label":"Copy object", "param":"s3://BUCKET1/OBJECT1 s3://BUCKET2[/OBJECT2]", "func":cmd_cp, "argc":2},
1525 {"cmd":"mv", "label":"Move object", "param":"s3://BUCKET1/OBJECT1 s3://BUCKET2[/OBJECT2]", "func":cmd_mv, "argc":2},
1526 {"cmd":"setacl", "label":"Modify Access control list for Bucket or Files", "param":"s3://BUCKET[/OBJECT]", "func":cmd_setacl, "argc":1},
1527 {"cmd":"accesslog", "label":"Enable/disable bucket access logging", "param":"s3://BUCKET", "func":cmd_accesslog, "argc":1},
1528 {"cmd":"sign", "label":"Sign arbitrary string using the secret key", "param":"STRING-TO-SIGN", "func":cmd_sign, "argc":1},
1529 {"cmd":"fixbucket", "label":"Fix invalid file names in a bucket", "param":"s3://BUCKET[/PREFIX]", "func":cmd_fixbucket, "argc":1},
1530
1531 ## Website commands
1532 {"cmd":"ws-create", "label":"Create Website from bucket", "param":"s3://BUCKET", "func":cmd_website_create, "argc":1},
1533 {"cmd":"ws-delete", "label":"Delete Website", "param":"s3://BUCKET", "func":cmd_website_delete, "argc":1},
1534 {"cmd":"ws-info", "label":"Info about Website", "param":"s3://BUCKET", "func":cmd_website_info, "argc":1},
1535
1536 ## CloudFront commands
1537 {"cmd":"cflist", "label":"List CloudFront distribution points", "param":"", "func":CfCmd.info, "argc":0},
1538 {"cmd":"cfinfo", "label":"Display CloudFront distribution point parameters", "param":"[cf://DIST_ID]", "func":CfCmd.info, "argc":0},
1539 {"cmd":"cfcreate", "label":"Create CloudFront distribution point", "param":"s3://BUCKET", "func":CfCmd.create, "argc":1},
1540 {"cmd":"cfdelete", "label":"Delete CloudFront distribution point", "param":"cf://DIST_ID", "func":CfCmd.delete, "argc":1},
1541 {"cmd":"cfmodify", "label":"Change CloudFront distribution point parameters", "param":"cf://DIST_ID", "func":CfCmd.modify, "argc":1},
1542 #{"cmd":"cfinval", "label":"Invalidate CloudFront objects", "param":"s3://BUCKET/OBJECT [s3://BUCKET/OBJECT ...]", "func":CfCmd.invalidate, "argc":1},
1543 {"cmd":"cfinvalinfo", "label":"Display CloudFront invalidation request(s) status", "param":"cf://DIST_ID[/INVAL_ID]", "func":CfCmd.invalinfo, "argc":1},
1544 ]
3cc025a @mludvig Renamed s3.py to s3cmd
mludvig authored
1545
ccb7853 @mludvig * Merged CloudFront support from branches/s3cmd-airlock
mludvig authored
1546 def format_commands(progname, commands_list):
d439efb @mludvig ATTENTION -- Mega WhiteSpace conversion !!!
mludvig authored
1547 help = "Commands:\n"
1548 for cmd in commands_list:
1549 help += " %s\n %s %s %s\n" % (cmd["label"], progname, cmd["cmd"], cmd["param"])
1550 return help
f4555c3 @mludvig - Use prefix for listings if specified.
mludvig authored
1551
9b7618a @mludvig Major rework of Config class:
mludvig authored
1552 class OptionMimeType(Option):
d439efb @mludvig ATTENTION -- Mega WhiteSpace conversion !!!
mludvig authored
1553 def check_mimetype(option, opt, value):
615eed4 @mludvig Allow optional parameters in --mime-type
mludvig authored
1554 if re.compile("^[a-z0-9]+/[a-z0-9+\.-]+(;.*)?$", re.IGNORECASE).match(value):
d439efb @mludvig ATTENTION -- Mega WhiteSpace conversion !!!
mludvig authored
1555 return value
1556 raise OptionValueError("option %s: invalid MIME-Type format: %r" % (opt, value))
9b7618a @mludvig Major rework of Config class:
mludvig authored
1557
4f11bf5 @mludvig * s3cmd: Pre-parse ACL parameters in OptionS3ACL()
mludvig authored
1558 class OptionS3ACL(Option):
d439efb @mludvig ATTENTION -- Mega WhiteSpace conversion !!!
mludvig authored
1559 def check_s3acl(option, opt, value):
1560 permissions = ('read', 'write', 'read_acp', 'write_acp', 'full_control', 'all')
1561 try:
1562 permission, grantee = re.compile("^(\w+):(.+)$", re.IGNORECASE).match(value).groups()
1563 if not permission or not grantee:
1564 raise
1565 if permission in permissions:
1566 return { 'name' : grantee, 'permission' : permission.upper() }
1567 else:
1568 raise OptionValueError("option %s: invalid S3 ACL permission: %s (valid values: %s)" %
1569 (opt, permission, ", ".join(permissions)))
1570 except:
1571 raise OptionValueError("option %s: invalid S3 ACL format: %r" % (opt, value))
4f11bf5 @mludvig * s3cmd: Pre-parse ACL parameters in OptionS3ACL()
mludvig authored
1572
1573 class OptionAll(OptionMimeType, OptionS3ACL):
d439efb @mludvig ATTENTION -- Mega WhiteSpace conversion !!!
mludvig authored
1574 TYPE_CHECKER = copy(Option.TYPE_CHECKER)
1575 TYPE_CHECKER["mimetype"] = OptionMimeType.check_mimetype
1576 TYPE_CHECKER["s3acl"] = OptionS3ACL.check_s3acl
1577 TYPES = Option.TYPES + ("mimetype", "s3acl")
9b7618a @mludvig Major rework of Config class:
mludvig authored
1578
f4555c3 @mludvig - Use prefix for listings if specified.
mludvig authored
1579 class MyHelpFormatter(IndentedHelpFormatter):
d439efb @mludvig ATTENTION -- Mega WhiteSpace conversion !!!
mludvig authored
1580 def format_epilog(self, epilog):
1581 if epilog:
1582 return "\n" + epilog + "\n"
1583 else:
1584 return ""
f4555c3 @mludvig - Use prefix for listings if specified.
mludvig authored
1585
4a52baa @mludvig * s3cmd: Wrapped all execution in a try/except block
mludvig authored
1586 def main():
d439efb @mludvig ATTENTION -- Mega WhiteSpace conversion !!!
mludvig authored
1587 global cfg
1588
1589 commands_list = get_commands_list()
1590 commands = {}
1591
1592 ## Populate "commands" from "commands_list"
1593 for cmd in commands_list:
1594 if cmd.has_key("cmd"):
1595 commands[cmd["cmd"]] = cmd
1596
1597 default_verbosity = Config().verbosity
1598 optparser = OptionParser(option_class=OptionAll, formatter=MyHelpFormatter())
1599 #optparser.disable_interspersed_args()
1600
1601 config_file = None
1602 if os.getenv("HOME"):
1603 config_file = os.path.join(os.getenv("HOME"), ".s3cfg")
1604 elif os.name == "nt" and os.getenv("USERPROFILE"):
1605 config_file = os.path.join(os.getenv("USERPROFILE").decode('mbcs'), "Application Data", "s3cmd.ini")
1606
1607 preferred_encoding = locale.getpreferredencoding() or "UTF-8"
1608
1609 optparser.set_defaults(encoding = preferred_encoding)
1610 optparser.set_defaults(config = config_file)
1611 optparser.set_defaults(verbosity = default_verbosity)
1612
3234ccf @mludvig Small improvements for --configure s3://bucket
mludvig authored
1613 optparser.add_option( "--configure", dest="run_configure", action="store_true", help="Invoke interactive (re)configuration tool. Optionally use as '--configure s3://come-bucket' to test access to a specific bucket instead of attempting to list them all.")
d439efb @mludvig ATTENTION -- Mega WhiteSpace conversion !!!
mludvig authored
1614 optparser.add_option("-c", "--config", dest="config", metavar="FILE", help="Config file name. Defaults to %default")
1615 optparser.add_option( "--dump-config", dest="dump_config", action="store_true", help="Dump current configuration after parsing config files and command line options and exit.")
1616
1617 optparser.add_option("-n", "--dry-run", dest="dry_run", action="store_true", help="Only show what should be uploaded or downloaded but don't actually do it. May still perform S3 requests to get bucket listings and other information though (only for file transfer commands)")
1618
1619 optparser.add_option("-e", "--encrypt", dest="encrypt", action="store_true", help="Encrypt files before uploading to S3.")
1620 optparser.add_option( "--no-encrypt", dest="encrypt", action="store_false", help="Don't encrypt files.")
1621 optparser.add_option("-f", "--force", dest="force", action="store_true", help="Force overwrite and other dangerous operations.")
1622 optparser.add_option( "--continue", dest="get_continue", action="store_true", help="Continue getting a partially downloaded file (only for [get] command).")
1623 optparser.add_option( "--skip-existing", dest="skip_existing", action="store_true", help="Skip over files that exist at the destination (only for [get] and [sync] commands).")
1624 optparser.add_option("-r", "--recursive", dest="recursive", action="store_true", help="Recursive upload, download or removal.")
1625 optparser.add_option( "--check-md5", dest="check_md5", action="store_true", help="Check MD5 sums when comparing files for [sync]. (default)")
1626 optparser.add_option( "--no-check-md5", dest="check_md5", action="store_false", help="Do not check MD5 sums when comparing files for [sync]. Only size will be compared. May significantly speed up transfer but may also miss some changed files.")
1627 optparser.add_option("-P", "--acl-public", dest="acl_public", action="store_true", help="Store objects with ACL allowing read for anyone.")
1628 optparser.add_option( "--acl-private", dest="acl_public", action="store_false", help="Store objects with default ACL allowing access for you only.")
1629 optparser.add_option( "--acl-grant", dest="acl_grants", type="s3acl", action="append", metavar="PERMISSION:EMAIL or USER_CANONICAL_ID", help="Grant stated permission to a given amazon user. Permission is one of: read, write, read_acp, write_acp, full_control, all")
1630 optparser.add_option( "--acl-revoke", dest="acl_revokes", type="s3acl", action="append", metavar="PERMISSION:USER_CANONICAL_ID", help="Revoke stated permission for a given amazon user. Permission is one of: read, write, read_acp, wr ite_acp, full_control, all")
1631
1632 optparser.add_option( "--delete-removed", dest="delete_removed", action="store_true", help="Delete remote objects with no corresponding local file [sync]")
1633 optparser.add_option( "--no-delete-removed", dest="delete_removed", action="store_false", help="Don't delete remote objects.")
d5dc2c0 @mdomsch add --delete-after option for sync
authored
1634 optparser.add_option( "--delete-after", dest="delete_after", action="store_true", help="Perform deletes after new uploads [sync]")
c3deb6a @mdomsch add --delay-updates option
authored
1635 optparser.add_option( "--delay-updates", dest="delay_updates", action="store_true", help="Put all updated files into place at end [sync]")
07c9e2d @mdomsch sync: add --add-destination, parallelize uploads to multiple destination...
authored
1636 optparser.add_option( "--add-destination", dest="additional_destinations", action="append", help="Additional destination for parallel uploads, in addition to last arg. May be repeated.")
d439efb @mludvig ATTENTION -- Mega WhiteSpace conversion !!!
mludvig authored
1637 optparser.add_option("-p", "--preserve", dest="preserve_attrs", action="store_true", help="Preserve filesystem attributes (mode, ownership, timestamps). Default for [sync] command.")
1638 optparser.add_option( "--no-preserve", dest="preserve_attrs", action="store_false", help="Don't store FS attributes")
1639 optparser.add_option( "--exclude", dest="exclude", action="append", metavar="GLOB", help="Filenames and paths matching GLOB will be excluded from sync")
1640 optparser.add_option( "--exclude-from", dest="exclude_from", action="append", metavar="FILE", help="Read --exclude GLOBs from FILE")
1641 optparser.add_option( "--rexclude", dest="rexclude", action="append", metavar="REGEXP", help="Filenames and paths matching REGEXP (regular expression) will be excluded from sync")
1642 optparser.add_option( "--rexclude-from", dest="rexclude_from", action="append", metavar="FILE", help="Read --rexclude REGEXPs from FILE")
1643 optparser.add_option( "--include", dest="include", action="append", metavar="GLOB", help="Filenames and paths matching GLOB will be included even if previously excluded by one of --(r)exclude(-from) patterns")
1644 optparser.add_option( "--include-from", dest="include_from", action="append", metavar="FILE", help="Read --include GLOBs from FILE")
1645 optparser.add_option( "--rinclude", dest="rinclude", action="append", metavar="REGEXP", help="Same as --include but uses REGEXP (regular expression) instead of GLOB")
1646 optparser.add_option( "--rinclude-from", dest="rinclude_from", action="append", metavar="FILE", help="Read --rinclude REGEXPs from FILE")
1647
335a9b9 @res0nat0r Added all bucket endpoints
res0nat0r authored
1648 optparser.add_option( "--bucket-location", dest="bucket_location", help="Datacentre to create bucket in. As of now the datacenters are: US (default), EU, ap-northeast-1, ap-southeast-1, sa-east-1, us-west-1 and us-west-2")
d439efb @mludvig ATTENTION -- Mega WhiteSpace conversion !!!
mludvig authored
1649 optparser.add_option( "--reduced-redundancy", "--rr", dest="reduced_redundancy", action="store_true", help="Store object with 'Reduced redundancy'. Lower per-GB price. [put, cp, mv]")
1650
1651 optparser.add_option( "--access-logging-target-prefix", dest="log_target_prefix", help="Target prefix for access logs (S3 URI) (for [cfmodify] and [accesslog] commands)")
1652 optparser.add_option( "--no-access-logging", dest="log_target_prefix", action="store_false", help="Disable access logging (for [cfmodify] and [accesslog] commands)")
1653
35612e6 @mludvig * Force MIME type with --mime-type=abc/xyz
mludvig authored
1654 optparser.add_option( "--default-mime-type", dest="default_mime_type", action="store_true", help="Default MIME-type for stored objects. Application default is binary/octet-stream.")
0d477b9 @mludvig MIME-Type guessing is now on by default
mludvig authored
1655 optparser.add_option( "--guess-mime-type", dest="guess_mime_type", action="store_true", help="Guess MIME-type of files by their extension or mime magic. Fall back to default MIME-Type as specified by --default-mime-type option")
35612e6 @mludvig * Force MIME type with --mime-type=abc/xyz
mludvig authored
1656 optparser.add_option( "--no-guess-mime-type", dest="guess_mime_type", action="store_false", help="Don't guess MIME-type and use the default type instead.")
1657 optparser.add_option("-m", "--mime-type", dest="mime_type", type="mimetype", metavar="MIME/TYPE", help="Force MIME-type. Override both --default-mime-type and --guess-mime-type.")
d439efb @mludvig ATTENTION -- Mega WhiteSpace conversion !!!
mludvig authored
1658
1659 optparser.add_option( "--add-header", dest="add_header", action="append", metavar="NAME:VALUE", help="Add a given HTTP header to the upload request. Can be used multiple times. For instance set 'Expires' or 'Cache-Control' headers (or both) using this options if you like.")
1660
1661 optparser.add_option( "--encoding", dest="encoding", metavar="ENCODING", help="Override autodetected terminal and filesystem encoding (character set). Autodetected: %s" % preferred_encoding)
1662 optparser.add_option( "--verbatim", dest="urlencoding_mode", action="store_const", const="verbatim", help="Use the S3 name as given on the command line. No pre-processing, encoding, etc. Use with caution!")
1663
589be07 @mludvig Fixed help text
mludvig authored
1664 optparser.add_option( "--disable-multipart", dest="enable_multipart", action="store_false", help="Disable multipart upload on files bigger than --multipart-chunk-size-mb")
8031016 @mludvig Renamed multipart_chunk_size to multipart_chunk_size_mb
mludvig authored
1665 optparser.add_option( "--multipart-chunk-size-mb", dest="multipart_chunk_size_mb", type="int", action="store", metavar="SIZE", help="Size of each chunk of a multipart upload. Files bigger than SIZE are automatically uploaded as multithreaded-multipart, smaller files are uploaded using the traditional method. SIZE is in Mega-Bytes, default chunk size is %defaultMB, minimum allowed chunk size is 5MB, maximum is 5GB.")
880e0de @mludvig Cleanup: pass enable_multipart via Config()
mludvig authored
1666
d439efb @mludvig ATTENTION -- Mega WhiteSpace conversion !!!
mludvig authored
1667 optparser.add_option( "--list-md5", dest="list_md5", action="store_true", help="Include MD5 sums in bucket listings (only for 'ls' command).")
1668 optparser.add_option("-H", "--human-readable-sizes", dest="human_readable_sizes", action="store_true", help="Print sizes in human readable form (eg 1kB instead of 1234).")
1669
1670 optparser.add_option( "--ws-index", dest="website_index", action="store", help="Name of error-document (only for [ws-create] command)")
1671 optparser.add_option( "--ws-error", dest="website_error", action="store", help="Name of index-document (only for [ws-create] command)")
1672
1673 optparser.add_option( "--progress", dest="progress_meter", action="store_true", help="Display progress meter (default on TTY).")
1674 optparser.add_option( "--no-progress", dest="progress_meter", action="store_false", help="Don't display progress meter (default on non-TTY).")
1675 optparser.add_option( "--enable", dest="enable", action="store_true", help="Enable given CloudFront distribution (only for [cfmodify] command)")
1676 optparser.add_option( "--disable", dest="enable", action="store_false", help="Enable given CloudFront distribution (only for [cfmodify] command)")
1677 optparser.add_option( "--cf-invalidate", dest="invalidate_on_cf", action="store_true", help="Invalidate the uploaded filed in CloudFront. Also see [cfinval] command.")
1678 optparser.add_option( "--cf-add-cname", dest="cf_cnames_add", action="append", metavar="CNAME", help="Add given CNAME to a CloudFront distribution (only for [cfcreate] and [cfmodify] commands)")
1679 optparser.add_option( "--cf-remove-cname", dest="cf_cnames_remove", action="append", metavar="CNAME", help="Remove given CNAME from a CloudFront distribution (only for [cfmodify] command)")
1680 optparser.add_option( "--cf-comment", dest="cf_comment", action="store", metavar="COMMENT", help="Set COMMENT for a given CloudFront distribution (only for [cfcreate] and [cfmodify] commands)")
1681 optparser.add_option( "--cf-default-root-object", dest="cf_default_root_object", action="store", metavar="DEFAULT_ROOT_OBJECT", help="Set the default root object to return when no object is specified in the URL. Use a relative path, i.e. default/index.html instead of /default/index.html or s3://bucket/default/index.html (only for [cfcreate] and [cfmodify] commands)")
1682 optparser.add_option("-v", "--verbose", dest="verbosity", action="store_const", const=logging.INFO, help="Enable verbose output.")
1683 optparser.add_option("-d", "--debug", dest="verbosity", action="store_const", const=logging.DEBUG, help="Enable debug output.")
1684 optparser.add_option( "--version", dest="show_version", action="store_true", help="Show s3cmd version (%s) and exit." % (PkgInfo.version))
1685 optparser.add_option("-F", "--follow-symlinks", dest="follow_symlinks", action="store_true", default=False, help="Follow symbolic links as if they are regular files")
488c956 @mdomsch add local tree MD5 caching
authored
1686 optparser.add_option( "--cache-file", dest="cache_file", action="store", default="", metavar="FILE", help="Cache FILE containing local source MD5 values")
d439efb @mludvig ATTENTION -- Mega WhiteSpace conversion !!!
mludvig authored
1687
1688 optparser.set_usage(optparser.usage + " COMMAND [parameters]")
1689 optparser.set_description('S3cmd is a tool for managing objects in '+
1690 'Amazon S3 storage. It allows for making and removing '+
1691 '"buckets" and uploading, downloading and removing '+
1692 '"objects" from these buckets.')
1693 optparser.epilog = format_commands(optparser.get_prog_name(), commands_list)
1694 optparser.epilog += ("\nFor more informations see the progect homepage:\n%s\n" % PkgInfo.url)
1695 optparser.epilog += ("\nConsider a donation if you have found s3cmd useful:\n%s/donate\n" % PkgInfo.url)
1696
1697 (options, args) = optparser.parse_args()
1698
1699 ## Some mucking with logging levels to enable
1700 ## debugging/verbose output for config file parser on request
1701 logging.basicConfig(level=options.verbosity,
1702 format='%(levelname)s: %(message)s',
1703 stream = sys.stderr)
1704
1705 if options.show_version:
1706 output(u"s3cmd version %s" % PkgInfo.version)
1707 sys.exit(0)
1708
1709 ## Now finally parse the config file
1710 if not options.config:
1711 error(u"Can't find a config file. Please use --config option.")
1712 sys.exit(1)
1713
1714 try:
1715 cfg = Config(options.config)
1716 except IOError, e:
1717 if options.run_configure:
1718 cfg = Config()
1719 else:
1720 error(u"%s: %s" % (options.config, e.strerror))
1721 error(u"Configuration file not available.")
1722 error(u"Consider using --configure parameter to create one.")
1723 sys.exit(1)
1724
1725 ## And again some logging level adjustments
1726 ## according to configfile and command line parameters
1727 if options.verbosity != default_verbosity:
1728 cfg.verbosity = options.verbosity
1729 logging.root.setLevel(cfg.verbosity)
1730
1731 ## Default to --progress on TTY devices, --no-progress elsewhere
1732 ## Can be overriden by actual --(no-)progress parameter
1733 cfg.update_option('progress_meter', sys.stdout.isatty())
1734
1735 ## Unsupported features on Win32 platform
1736 if os.name == "nt":
1737 if cfg.preserve_attrs:
1738 error(u"Option --preserve is not yet supported on MS Windows platform. Assuming --no-preserve.")
1739 cfg.preserve_attrs = False
1740 if cfg.progress_meter:
1741 error(u"Option --progress is not yet supported on MS Windows platform. Assuming --no-progress.")
1742 cfg.p