Skip to content
This repository

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse code

* s3cmd: Support for recursive [cp] and [mv], including

  multiple-source arguments, --include/--exclude,
  --dry-run, etc.
* run-tests.py: Tests for the above.
* S3/S3.py: Preserve metadata (eg ACL or MIME type) 
  during [cp] and [mv].
* NEWS, TODO: Updated.



git-svn-id: https://s3tools.svn.sourceforge.net/svnroot/s3tools/s3cmd/trunk@389 830e0280-6d2a-0410-9c65-932aecc39d9d
  • Loading branch information...
commit e0b946c028d790956a27813a7adc59863bd72719 1 parent 3677a3b
Michal Ludvig authored May 27, 2009
10  ChangeLog
... ...
@@ -1,5 +1,15 @@
1 1
 2009-05-28  Michal Ludvig  <michal@logix.cz>
2 2
 
  3
+	* s3cmd: Support for recursive [cp] and [mv], including
  4
+	  multiple-source arguments, --include/--exclude,
  5
+	  --dry-run, etc.
  6
+	* run-tests.py: Tests for the above.
  7
+	* S3/S3.py: Preserve metadata (eg ACL or MIME type) 
  8
+	  during [cp] and [mv].
  9
+	* NEWS, TODO: Updated.
  10
+
  11
+2009-05-28  Michal Ludvig  <michal@logix.cz>
  12
+
3 13
 	* run-tests.py: Added --verbose mode.
4 14
 
5 15
 2009-05-27  Michal Ludvig  <michal@logix.cz>
6  NEWS
... ...
@@ -1,11 +1,13 @@
1 1
 s3cmd 1.0.0
2 2
 ===========
3  
-* New command 'sign' for signing for instance
4  
-  the POST upload policies.
  3
+* New command 'sign' for signing e.g. POST upload policies.
5 4
 * Fixed handling of filenames that differ only in 
6 5
   capitalisation (eg blah.txt vs Blah.TXT).
7 6
 * Added --verbatim mode, preventing most filenames 
8 7
   pre-processing. Good for fixing unreadable buckets.
  8
+* Added --recursive support for [cp] and [mv], including
  9
+  multiple-source arguments, --include/--exclude, --dry-run, etc.
  10
+
9 11
 
10 12
 s3cmd 0.9.9   -   2009-02-17
11 13
 ===========
6  S3/S3.py
@@ -275,10 +275,12 @@ def object_copy(self, src_uri, dst_uri, extra_headers = None):
275 275
 			raise ValueError("Expected URI type 's3', got '%s'" % dst_uri.type)
276 276
 		headers = SortedDict(ignore_case = True)
277 277
 		headers['x-amz-copy-source'] = "/%s/%s" % (src_uri.bucket(), self.urlencode_string(src_uri.object()))
  278
+		## TODO: For now COPY, later maybe add a switch?
  279
+		headers['x-amz-metadata-directive'] = "COPY"
278 280
 		if self.config.acl_public:
279 281
 			headers["x-amz-acl"] = "public-read"
280  
-		if extra_headers:
281  
-			headers.update(extra_headers)
  282
+		# if extra_headers:
  283
+		# 	headers.update(extra_headers)
282 284
 		request = self.create_request("OBJECT_PUT", uri = dst_uri, headers = headers)
283 285
 		response = self.send_request(request)
284 286
 		return response
1  TODO
@@ -6,7 +6,6 @@ TODO list for s3cmd project
6 6
     (at the moment it'll always download).
7 7
   - Enable --exclude for [del], [setacl], [ls].
8 8
   - Enable --dry-run for [del], [setacl], reject for all others.
9  
-  - Recursive cp/mv on remote "folders".
10 9
   - Allow change /tmp to somewhere else
11 10
   - With --guess-mime use 'magic' module if available.
12 11
   - Support --preserve for [put] and [get]. Update manpage.
39  run-tests.py
@@ -344,15 +344,44 @@ def test_flushdir(label, dir_name):
344 344
 test_s3cmd("Get multiple files", ['get', 's3://s3cmd-autotest-1/xyz/etc2/Logo.PNG', 's3://s3cmd-autotest-1/xyz/etc/AtomicClockRadio.ttf', 'testsuite-out'],
345 345
 	must_find = [ u"saved as 'testsuite-out/Logo.PNG'", u"saved as 'testsuite-out/AtomicClockRadio.ttf'" ])
346 346
 
347  
-
348  
-## ====== Copy between buckets
349  
-test_s3cmd("Copy between buckets", ['cp', 's3://s3cmd-autotest-1/xyz/etc2/Logo.PNG', 's3://s3cmd-Autotest-3'],
350  
-	must_find = [ "File s3://s3cmd-autotest-1/xyz/etc2/Logo.PNG copied to s3://s3cmd-Autotest-3/xyz/etc2/Logo.PNG" ])
351  
-
352 347
 ## ====== Upload files differing in capitalisation
353 348
 test_s3cmd("blah.txt / Blah.txt", ['put', '-r', 'testsuite/blahBlah', 's3://s3cmd-autotest-1/'],
354 349
 	must_find = [ 's3://s3cmd-autotest-1/blahBlah/Blah.txt', 's3://s3cmd-autotest-1/blahBlah/blah.txt' ])
355 350
 
  351
+## ====== Copy between buckets
  352
+test_s3cmd("Copy between buckets", ['cp', 's3://s3cmd-autotest-1/xyz/etc2/Logo.PNG', 's3://s3cmd-Autotest-3/xyz/etc2/logo.png'],
  353
+	must_find = [ "File s3://s3cmd-autotest-1/xyz/etc2/Logo.PNG copied to s3://s3cmd-Autotest-3/xyz/etc2/logo.png" ])
  354
+
  355
+## ====== Recursive copy
  356
+test_s3cmd("Recursive copy, set ACL", ['cp', '-r', '--acl-public', 's3://s3cmd-autotest-1/xyz/', 's3://s3cmd-autotest-2/copy', '--exclude', '.svn/*'],
  357
+	must_find = [ "File s3://s3cmd-autotest-1/xyz/etc2/Logo.PNG copied to s3://s3cmd-autotest-2/copy/etc2/Logo.PNG",
  358
+	              "File s3://s3cmd-autotest-1/xyz/blahBlah/Blah.txt copied to s3://s3cmd-autotest-2/copy/blahBlah/Blah.txt",
  359
+	              "File s3://s3cmd-autotest-1/xyz/blahBlah/blah.txt copied to s3://s3cmd-autotest-2/copy/blahBlah/blah.txt" ],
  360
+	must_not_find = [ ".svn" ])
  361
+
  362
+## ====== Verify ACL and MIME type
  363
+test_s3cmd("Verify ACL and MIME type", ['info', 's3://s3cmd-autotest-2/copy/etc2/Logo.PNG' ],
  364
+	must_find_re = [ "MIME type:.*image/png", 
  365
+	                 "ACL:.*\*anon\*: READ",
  366
+					 "URL:.*http://s3cmd-autotest-2.s3.amazonaws.com/copy/etc2/Logo.PNG" ])
  367
+
  368
+## ====== Multi source move
  369
+test_s3cmd("Multi-source move", ['mv', '-r', 's3://s3cmd-autotest-2/copy/blahBlah/Blah.txt', 's3://s3cmd-autotest-2/copy/etc/', 's3://s3cmd-autotest-2/moved/'],
  370
+	must_find = [ "File s3://s3cmd-autotest-2/copy/blahBlah/Blah.txt moved to s3://s3cmd-autotest-2/moved/Blah.txt",
  371
+	              "File s3://s3cmd-autotest-2/copy/etc/AtomicClockRadio.ttf moved to s3://s3cmd-autotest-2/moved/AtomicClockRadio.ttf",
  372
+				  "File s3://s3cmd-autotest-2/copy/etc/TypeRa.ttf moved to s3://s3cmd-autotest-2/moved/TypeRa.ttf" ],
  373
+	must_not_find = [ "blah.txt" ])
  374
+
  375
+## ====== Verify move
  376
+test_s3cmd("Verify move", ['ls', '-r', 's3://s3cmd-autotest-2'],
  377
+	must_find = [ "s3://s3cmd-autotest-2/moved/Blah.txt",
  378
+	              "s3://s3cmd-autotest-2/moved/AtomicClockRadio.ttf",
  379
+				  "s3://s3cmd-autotest-2/moved/TypeRa.ttf",
  380
+				  "s3://s3cmd-autotest-2/copy/blahBlah/blah.txt" ],
  381
+	must_not_find = [ "s3://s3cmd-autotest-2/copy/blahBlah/Blah.txt",
  382
+					  "s3://s3cmd-autotest-2/copy/etc/AtomicClockRadio.ttf",
  383
+					  "s3://s3cmd-autotest-2/copy/etc/TypeRa.ttf" ])
  384
+
356 385
 ## ====== Simple delete
357 386
 test_s3cmd("Simple delete", ['del', 's3://s3cmd-autotest-1/xyz/etc2/Logo.PNG'],
358 387
 	must_find = [ "File s3://s3cmd-autotest-1/xyz/etc2/Logo.PNG deleted" ])
64  s3cmd
@@ -513,32 +513,64 @@ def subcmd_object_del_uri(uri, recursive = None):
513 513
 		response = s3.object_delete(_uri)
514 514
 		output(u"File %s deleted" % _uri)
515 515
 
516  
-def subcmd_cp_mv(args, process_fce, message):
517  
-	src_uri = S3Uri(args.pop(0))
518  
-	dst_uri = S3Uri(args.pop(0))
  516
+def subcmd_cp_mv(args, process_fce, action_str, message):
  517
+	if len(args) < 2:
  518
+		raise ParameterError("Expecting two or more S3 URIs for " + action_str)
  519
+	dst_base_uri = S3Uri(args.pop())
  520
+	if dst_base_uri.type != "s3":
  521
+		raise ParameterError("Destination must be S3 URI. To download a file use 'get' or 'sync'.")
  522
+	destination_base = dst_base_uri.uri()
519 523
 
520  
-	if len(args): 
521  
-		raise ParameterError("Too many parameters! Expected: %s" % commands['cp']['param']) 
  524
+	remote_list = fetch_remote_list(args, require_attribs = False)
  525
+	remote_list, exclude_list = _filelist_filter_exclude_include(remote_list)
  526
+
  527
+	remote_count = len(remote_list)
522 528
 
523  
-	if src_uri.type != "s3" or dst_uri.type != "s3": 
524  
-		raise ParameterError("Parameters are not URIs! Expected: %s" % commands['cp']['param']) 
  529
+	info(u"Summary: %d remote files to %s" % (remote_count, action_str))
525 530
 
526  
-	if dst_uri.object() == "":
527  
-		dst_uri = S3Uri(dst_uri.uri() + src_uri.object())
  531
+	if cfg.recursive:
  532
+		if not destination_base.endswith("/"):
  533
+			destination_base += "/"
  534
+		for key in remote_list:
  535
+			remote_list[key]['dest_name'] = destination_base + key
  536
+	else:
  537
+		key = remote_list.keys()[0]
  538
+		if destination_base.endswith("/"):
  539
+			remote_list[key]['dest_name'] = destination_base + key
  540
+		else:
  541
+			remote_list[key]['dest_name'] = destination_base
  542
+
  543
+	if cfg.dry_run:
  544
+		for key in exclude_list:
  545
+			output(u"exclude: %s" % unicodise(key))
  546
+		for key in remote_list:
  547
+			output(u"%s: %s -> %s" % (action_str, remote_list[key]['object_uri_str'], remote_list[key]['dest_name']))
528 548
 
529  
-	extra_headers = copy(cfg.extra_headers)
530  
-	response = process_fce(src_uri, dst_uri, extra_headers) 
531  
-	output(message % { "src" : src_uri, "dst" : dst_uri})
532  
-	if Config().acl_public:
533  
-		output(u"Public URL is: %s" % dst_uri.public_url())
  549
+		warning(u"Exitting now because of --dry-run")
  550
+		return
  551
+
  552
+	seq = 0
  553
+	for key in remote_list:
  554
+		seq += 1
  555
+		seq_label = "[%d of %d]" % (seq, remote_count)
  556
+
  557
+		item = remote_list[key]
  558
+		src_uri = S3Uri(item['object_uri_str'])
  559
+		dst_uri = S3Uri(item['dest_name'])
  560
+
  561
+		extra_headers = copy(cfg.extra_headers)
  562
+		response = process_fce(src_uri, dst_uri, extra_headers) 
  563
+		output(message % { "src" : src_uri, "dst" : dst_uri })
  564
+		if Config().acl_public:
  565
+			info(u"Public URL is: %s" % dst_uri.public_url())
534 566
 
535 567
 def cmd_cp(args):
536 568
 	s3 = S3(Config())
537  
-	subcmd_cp_mv(args, s3.object_copy, "File %(src)s copied to %(dst)s")
  569
+	subcmd_cp_mv(args, s3.object_copy, "copy", "File %(src)s copied to %(dst)s")
538 570
 
539 571
 def cmd_mv(args):
540 572
 	s3 = S3(Config())
541  
-	subcmd_cp_mv(args, s3.object_move, "File %(src)s moved to %(dst)s")
  573
+	subcmd_cp_mv(args, s3.object_move, "move", "File %(src)s moved to %(dst)s")
542 574
 
543 575
 def cmd_info(args):
544 576
 	s3 = S3(Config())

0 notes on commit e0b946c

Please sign in to comment.
Something went wrong with that request. Please try again.