Skip to content
This repository

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse code

Option to follow local symlinks for sync and put (--follow-symlinks o…

…ption)

including tests and documentation.  Also addition to run-tests.py:
--bucket-prefix option, to allow different developers to run tests in their own
sandbox


git-svn-id: http://s3tools.svn.sourceforge.net/svnroot/s3tools/s3cmd/trunk@437 830e0280-6d2a-0410-9c65-932aecc39d9d
  • Loading branch information...
commit c6d507b1ab2bd9648a2614be121358960bdcabdf 1 parent e66cf50
authored July 26, 2010
7  ChangeLog
... ...
@@ -1,3 +1,10 @@
  1
+2010-07-25  Aaron Maxwell  <amax@resymbol.net>
  2
+
  3
+	* S3/Config.py, testsuite/etc/brokenlink.png, testsuite/etc/more/linked-dir, testsuite/etc/more/give-me-more.txt, testsuite/etc/linked.png, testsuite/etc/linked1.png, run-tests.py, s3cmd.1, s3cmd:
  4
+	  Option to follow local symlinks for sync and put (--follow-symlinks option), including tests and documentation
  5
+	* run-tests.py:
  6
+	  --bucket-prefix option, to allow different developers to run tests in their own sandbox
  7
+
1 8
 2010-07-08  Michal Ludvig  <mludvig@logix.net.nz>
2 9
 
3 10
 	* run-tests.py, testsuite/crappy-file-name.tar.gz:
1  S3/Config.py
@@ -74,6 +74,7 @@ class Config(object):
74 74
 	urlencoding_mode = "normal"
75 75
 	log_target_prefix = ""
76 76
 	reduced_redundancy = False
  77
+        follow_symlinks=False
77 78
 
78 79
 	## Creating a singleton
79 80
 	def __new__(self, configfile = None):
190  run-tests.py
@@ -48,7 +48,7 @@
48 48
 	have_encoding = os.path.isdir('testsuite/encodings/' + encoding)
49 49
 
50 50
 if have_encoding:
51  
-	enc_base_remote = "s3://s3cmd-autotest-1/xyz/%s/" % encoding
  51
+	#enc_base_remote = "%s/xyz/%s/" % (pbucket(1), encoding)
52 52
 	enc_pattern = patterns[encoding]
53 53
 else:
54 54
 	print encoding + " specific files not found."
@@ -170,9 +170,13 @@ def test_flushdir(label, dir_name):
170 170
 	test_rmdir(label + "(rm)", dir_name)
171 171
 	return test_mkdir(label + "(mk)", dir_name)
172 172
 
  173
+bucket_prefix = ''
173 174
 argv = sys.argv[1:]
174 175
 while argv:
175 176
 	arg = argv.pop(0)
  177
+        if arg.startswith('--bucket-prefix='):
  178
+                print "Usage: '--bucket-prefix PREFIX', not '--bucket-prefix=PREFIX'"
  179
+                sys.exit(0)
176 180
 	if arg in ("-h", "--help"):
177 181
 		print "%s A B K..O -N" % sys.argv[0]
178 182
 		print "Run tests number A, B and K through to O, except for N"
@@ -183,6 +187,13 @@ def test_flushdir(label, dir_name):
183 187
 	if arg in ("-v", "--verbose"):
184 188
 		verbose = True
185 189
 		continue
  190
+        if arg in ("-p", "--bucket-prefix"):
  191
+                try:
  192
+                        bucket_prefix = argv.pop(0)
  193
+                except IndexError:
  194
+                        print "Bucket prefix option must explicitly supply a bucket name prefix"
  195
+                        sys.exit(0)
  196
+                continue
186 197
 	if arg.find("..") >= 0:
187 198
 		range_idx = arg.find("..")
188 199
 		range_start = arg[:range_idx] or 0
@@ -196,66 +207,77 @@ def test_flushdir(label, dir_name):
196 207
 if not run_tests:
197 208
 	run_tests = range(0, 999)
198 209
 
  210
+# helper functions for generating bucket names
  211
+def bucket(tail):
  212
+        '''Test bucket name'''
  213
+        label = 'autotest'
  214
+        if str(tail) == '3':
  215
+                label = 'Autotest'
  216
+        return '%ss3cmd-%s-%s' % (bucket_prefix, label, tail)
  217
+def pbucket(tail):
  218
+        '''Like bucket(), but prepends "s3://" for you'''
  219
+        return 's3://' + bucket(tail)
  220
+
199 221
 ## ====== Remove test buckets
200  
-test_s3cmd("Remove test buckets", ['rb', '-r', 's3://s3cmd-autotest-1', 's3://s3cmd-autotest-2', 's3://s3cmd-Autotest-3'],
201  
-	must_find = [ "Bucket 's3://s3cmd-autotest-1/' removed",
202  
-		      "Bucket 's3://s3cmd-autotest-2/' removed",
203  
-		      "Bucket 's3://s3cmd-Autotest-3/' removed" ])
  222
+test_s3cmd("Remove test buckets", ['rb', '-r', pbucket(1), pbucket(2), pbucket(3)],
  223
+	must_find = [ "Bucket '%s/' removed" % pbucket(1),
  224
+		      "Bucket '%s/' removed" % pbucket(2),
  225
+		      "Bucket '%s/' removed" % pbucket(3) ])
204 226
 
205 227
 
206 228
 ## ====== Create one bucket (EU)
207  
-test_s3cmd("Create one bucket (EU)", ['mb', '--bucket-location=EU', 's3://s3cmd-autotest-1'], 
208  
-	must_find = "Bucket 's3://s3cmd-autotest-1/' created")
  229
+test_s3cmd("Create one bucket (EU)", ['mb', '--bucket-location=EU', pbucket(1)], 
  230
+	must_find = "Bucket '%s/' created" % pbucket(1))
209 231
 
210 232
 
211 233
 
212 234
 ## ====== Create multiple buckets
213  
-test_s3cmd("Create multiple buckets", ['mb', 's3://s3cmd-autotest-2', 's3://s3cmd-Autotest-3'], 
214  
-	must_find = [ "Bucket 's3://s3cmd-autotest-2/' created", "Bucket 's3://s3cmd-Autotest-3/' created" ])
  235
+test_s3cmd("Create multiple buckets", ['mb', pbucket(2), pbucket(3)], 
  236
+	must_find = [ "Bucket '%s/' created" % pbucket(2), "Bucket '%s/' created" % pbucket(3)])
215 237
 
216 238
 
217 239
 ## ====== Invalid bucket name
218  
-test_s3cmd("Invalid bucket name", ["mb", "--bucket-location=EU", "s3://s3cmd-Autotest-EU"], 
  240
+test_s3cmd("Invalid bucket name", ["mb", "--bucket-location=EU", pbucket('EU')], 
219 241
 	retcode = 1,
220  
-	must_find = "ERROR: Parameter problem: Bucket name 's3cmd-Autotest-EU' contains disallowed character", 
  242
+	must_find = "ERROR: Parameter problem: Bucket name '%s' contains disallowed character" % bucket('EU'), 
221 243
 	must_not_find_re = "Bucket.*created")
222 244
 
223 245
 
224 246
 ## ====== Buckets list
225 247
 test_s3cmd("Buckets list", ["ls"], 
226  
-	must_find = [ "autotest-1", "autotest-2", "Autotest-3" ], must_not_find_re = "Autotest-EU")
  248
+	must_find = [ "autotest-1", "autotest-2", "Autotest-3" ], must_not_find_re = "autotest-EU")
227 249
 
228 250
 
229 251
 ## ====== Sync to S3
230  
-test_s3cmd("Sync to S3", ['sync', 'testsuite/', 's3://s3cmd-autotest-1/xyz/', '--exclude', '.svn/*', '--exclude', '*.png', '--no-encrypt', '--exclude-from', 'testsuite/exclude.encodings' ],
  252
+test_s3cmd("Sync to S3", ['sync', 'testsuite/', pbucket(1) + '/xyz/', '--exclude', '.svn/*', '--exclude', '*.png', '--no-encrypt', '--exclude-from', 'testsuite/exclude.encodings' ],
231 253
 	must_find = [ "WARNING: 32 non-printable characters replaced in: crappy-file-name/too-crappy ^A^B^C^D^E^F^G^H^I^J^K^L^M^N^O^P^Q^R^S^T^U^V^W^X^Y^Z^[^\^]^^^_^? +-[\]^<>%%\"'#{}`&?.end",
232  
-	              "stored as 's3://s3cmd-autotest-1/xyz/crappy-file-name/too-crappy ^A^B^C^D^E^F^G^H^I^J^K^L^M^N^O^P^Q^R^S^T^U^V^W^X^Y^Z^[^\^]^^^_^? +-[\\]^<>%%\"'#{}`&?.end'" ],
  254
+	              "stored as '%s/xyz/crappy-file-name/too-crappy ^A^B^C^D^E^F^G^H^I^J^K^L^M^N^O^P^Q^R^S^T^U^V^W^X^Y^Z^[^\^]^^^_^? +-[\\]^<>%%%%\"'#{}`&?.end'" % pbucket(1) ],
233 255
 	must_not_find_re = [ "\.svn/", "\.png$" ])
234 256
 
235 257
 if have_encoding:
236 258
 	## ====== Sync UTF-8 / GBK / ... to S3
237  
-	test_s3cmd("Sync %s to S3" % encoding, ['sync', 'testsuite/encodings/' + encoding, 's3://s3cmd-autotest-1/xyz/encodings/', '--exclude', '.svn/*', '--no-encrypt' ],
238  
-		must_find = [ u"File 'testsuite/encodings/%(encoding)s/%(pattern)s' stored as 's3://s3cmd-autotest-1/xyz/encodings/%(encoding)s/%(pattern)s'" % { 'encoding' : encoding, 'pattern' : enc_pattern } ])
  259
+	test_s3cmd("Sync %s to S3" % encoding, ['sync', 'testsuite/encodings/' + encoding, '%s/xyz/encodings/' % pbucket(1), '--exclude', '.svn/*', '--no-encrypt' ],
  260
+		must_find = [ u"File 'testsuite/encodings/%(encoding)s/%(pattern)s' stored as '%(pbucket)s/xyz/encodings/%(encoding)s/%(pattern)s'" % { 'encoding' : encoding, 'pattern' : enc_pattern , 'pbucket' : pbucket(1)} ])
239 261
 
240 262
 
241 263
 ## ====== List bucket content
242  
-must_find_re = [ u"DIR   s3://s3cmd-autotest-1/xyz/binary/$", u"DIR   s3://s3cmd-autotest-1/xyz/etc/$" ]
  264
+must_find_re = [ u"DIR   %s/xyz/binary/$" % pbucket(1) , u"DIR   %s/xyz/etc/$" % pbucket(1) ]
243 265
 must_not_find = [ u"random-crap.md5", u".svn" ]
244  
-test_s3cmd("List bucket content", ['ls', 's3://s3cmd-autotest-1/xyz/'],
  266
+test_s3cmd("List bucket content", ['ls', '%s/xyz/' % pbucket(1) ],
245 267
 	must_find_re = must_find_re,
246 268
 	must_not_find = must_not_find)
247 269
 
248 270
 
249 271
 ## ====== List bucket recursive
250  
-must_find = [ u"s3://s3cmd-autotest-1/xyz/binary/random-crap.md5" ]
  272
+must_find = [ u"%s/xyz/binary/random-crap.md5" % pbucket(1) ]
251 273
 if have_encoding:
252  
-	must_find.append(u"s3://s3cmd-autotest-1/xyz/encodings/%(encoding)s/%(pattern)s" % { 'encoding' : encoding, 'pattern' : enc_pattern })
253  
-test_s3cmd("List bucket recursive", ['ls', '--recursive', 's3://s3cmd-autotest-1'],
  274
+	must_find.append(u"%(pbucket)s/xyz/encodings/%(encoding)s/%(pattern)s" % { 'encoding' : encoding, 'pattern' : enc_pattern, 'pbucket' : pbucket(1) })
  275
+test_s3cmd("List bucket recursive", ['ls', '--recursive', pbucket(1)],
254 276
 	must_find = must_find,
255 277
 	must_not_find = [ "logo.png" ])
256 278
 
257 279
 ## ====== FIXME
258  
-# test_s3cmd("Recursive put", ['put', '--recursive', 'testsuite/etc', 's3://s3cmd-autotest-1/xyz/'])
  280
+# test_s3cmd("Recursive put", ['put', '--recursive', 'testsuite/etc', '%s/xyz/' % pbucket(1) ])
259 281
 
260 282
 
261 283
 ## ====== Clean up local destination dir
@@ -263,10 +285,10 @@ def test_flushdir(label, dir_name):
263 285
 
264 286
 
265 287
 ## ====== Sync from S3
266  
-must_find = [ "File 's3://s3cmd-autotest-1/xyz/binary/random-crap.md5' stored as 'testsuite-out/xyz/binary/random-crap.md5'" ]
  288
+must_find = [ "File '%s/xyz/binary/random-crap.md5' stored as 'testsuite-out/xyz/binary/random-crap.md5'" % pbucket(1) ]
267 289
 if have_encoding:
268  
-	must_find.append(u"File 's3://s3cmd-autotest-1/xyz/encodings/%(encoding)s/%(pattern)s' stored as 'testsuite-out/xyz/encodings/%(encoding)s/%(pattern)s' " % { 'encoding' : encoding, 'pattern' : enc_pattern })
269  
-test_s3cmd("Sync from S3", ['sync', 's3://s3cmd-autotest-1/xyz', 'testsuite-out'],
  290
+	must_find.append(u"File '%(pbucket)s/xyz/encodings/%(encoding)s/%(pattern)s' stored as 'testsuite-out/xyz/encodings/%(encoding)s/%(pattern)s' " % { 'encoding' : encoding, 'pattern' : enc_pattern, 'pbucket' : pbucket(1) })
  291
+test_s3cmd("Sync from S3", ['sync', '%s/xyz' % pbucket(1), 'testsuite-out'],
270 292
 	must_find = must_find)
271 293
 
272 294
 
@@ -275,61 +297,63 @@ def test_flushdir(label, dir_name):
275 297
 
276 298
 
277 299
 ## ====== Put public, guess MIME
278  
-test_s3cmd("Put public, guess MIME", ['put', '--guess-mime-type', '--acl-public', 'testsuite/etc/logo.png', 's3://s3cmd-autotest-1/xyz/etc/logo.png'],
279  
-	must_find = [ "stored as 's3://s3cmd-autotest-1/xyz/etc/logo.png'" ])
  300
+test_s3cmd("Put public, guess MIME", ['put', '--guess-mime-type', '--acl-public', 'testsuite/etc/logo.png', '%s/xyz/etc/logo.png' % pbucket(1)],
  301
+	must_find = [ "stored as '%s/xyz/etc/logo.png'" % pbucket(1) ])
280 302
 
281 303
 
282 304
 ## ====== Retrieve from URL
283 305
 if have_wget:
284  
-	test("Retrieve from URL", ['wget', '-O', 'testsuite-out/logo.png', 'http://s3cmd-autotest-1.s3.amazonaws.com/xyz/etc/logo.png'],
  306
+	test("Retrieve from URL", ['wget', '-O', 'testsuite-out/logo.png', 'http://%s.s3.amazonaws.com/xyz/etc/logo.png' % bucket(1)],
285 307
 		must_find_re = [ 'logo.png.*saved \[22059/22059\]' ])
286 308
 
287 309
 
288 310
 ## ====== Change ACL to Private
289  
-test_s3cmd("Change ACL to Private", ['setacl', '--acl-private', 's3://s3cmd-autotest-1/xyz/etc/l*.png'],
  311
+test_s3cmd("Change ACL to Private", ['setacl', '--acl-private', '%s/xyz/etc/l*.png' % pbucket(1)],
290 312
 	must_find = [ "logo.png: ACL set to Private" ])
291 313
 
292 314
 
293 315
 ## ====== Verify Private ACL
294 316
 if have_wget:
295  
-	test("Verify Private ACL", ['wget', '-O', 'testsuite-out/logo.png', 'http://s3cmd-autotest-1.s3.amazonaws.com/xyz/etc/logo.png'],
  317
+	test("Verify Private ACL", ['wget', '-O', 'testsuite-out/logo.png', 'http://%s.s3.amazonaws.com/xyz/etc/logo.png' % bucket(1)],
296 318
 		retcode = 1,
297 319
 		must_find_re = [ 'ERROR 403: Forbidden' ])
298 320
 
299 321
 
300 322
 ## ====== Change ACL to Public
301  
-test_s3cmd("Change ACL to Public", ['setacl', '--acl-public', '--recursive', 's3://s3cmd-autotest-1/xyz/etc/', '-v'],
  323
+test_s3cmd("Change ACL to Public", ['setacl', '--acl-public', '--recursive', '%s/xyz/etc/' % pbucket(1) , '-v'],
302 324
 	must_find = [ "logo.png: ACL set to Public" ])
303 325
 
304 326
 
305 327
 ## ====== Verify Public ACL
306 328
 if have_wget:
307  
-	test("Verify Public ACL", ['wget', '-O', 'testsuite-out/logo.png', 'http://s3cmd-autotest-1.s3.amazonaws.com/xyz/etc/logo.png'],
  329
+	test("Verify Public ACL", ['wget', '-O', 'testsuite-out/logo.png', 'http://%s.s3.amazonaws.com/xyz/etc/logo.png' % bucket(1)],
308 330
 		must_find_re = [ 'logo.png.*saved \[22059/22059\]' ])
309 331
 
310 332
 
311 333
 ## ====== Sync more to S3
312  
-test_s3cmd("Sync more to S3", ['sync', 'testsuite/', 's3://s3cmd-autotest-1/xyz/', '--no-encrypt' ],
313  
-	must_find = [ "File 'testsuite/.svn/entries' stored as 's3://s3cmd-autotest-1/xyz/.svn/entries' " ])
  334
+test_s3cmd("Sync more to S3", ['sync', 'testsuite/', 's3://%s/xyz/' % bucket(1), '--no-encrypt' ],
  335
+	must_find = [ "File 'testsuite/.svn/entries' stored as '%s/xyz/.svn/entries' " % pbucket(1) ],
  336
+	must_not_find = [ "File 'testsuite/etc/linked.png' stored as '%s/xyz/etc/linked.png" % pbucket(1) ])
  337
+           
314 338
 
315 339
 
316 340
 ## ====== Rename within S3
317  
-test_s3cmd("Rename within S3", ['mv', 's3://s3cmd-autotest-1/xyz/etc/logo.png', 's3://s3cmd-autotest-1/xyz/etc2/Logo.PNG'],
318  
-	must_find = [ 'File s3://s3cmd-autotest-1/xyz/etc/logo.png moved to s3://s3cmd-autotest-1/xyz/etc2/Logo.PNG' ])
  341
+test_s3cmd("Rename within S3", ['mv', '%s/xyz/etc/logo.png' % pbucket(1), '%s/xyz/etc2/Logo.PNG' % pbucket(1)],
  342
+	must_find = [ 'File %s/xyz/etc/logo.png moved to %s/xyz/etc2/Logo.PNG' % (pbucket(1), pbucket(1))])
319 343
 
320 344
 
321 345
 ## ====== Rename (NoSuchKey)
322  
-test_s3cmd("Rename (NoSuchKey)", ['mv', 's3://s3cmd-autotest-1/xyz/etc/logo.png', 's3://s3cmd-autotest-1/xyz/etc2/Logo.PNG'],
  346
+test_s3cmd("Rename (NoSuchKey)", ['mv', '%s/xyz/etc/logo.png' % pbucket(1), '%s/xyz/etc2/Logo.PNG' % pbucket(1)],
323 347
 	retcode = 1,
324 348
 	must_find_re = [ 'ERROR:.*NoSuchKey' ],
325  
-	must_not_find = [ 'File s3://s3cmd-autotest-1/xyz/etc/logo.png moved to s3://s3cmd-autotest-1/xyz/etc2/Logo.PNG' ])
  349
+	must_not_find = [ 'File %s/xyz/etc/logo.png moved to %s/xyz/etc2/Logo.PNG' % (pbucket(1), pbucket(1)) ])
326 350
 
327 351
 
328 352
 ## ====== Sync more from S3
329  
-test_s3cmd("Sync more from S3", ['sync', '--delete-removed', 's3://s3cmd-autotest-1/xyz', 'testsuite-out'],
  353
+test_s3cmd("Sync more from S3", ['sync', '--delete-removed', '%s/xyz' % pbucket(1), 'testsuite-out'],
330 354
 	must_find = [ "deleted: testsuite-out/logo.png",
331  
-	              "File 's3://s3cmd-autotest-1/xyz/etc2/Logo.PNG' stored as 'testsuite-out/xyz/etc2/Logo.PNG' (22059 bytes", 
332  
-	              "File 's3://s3cmd-autotest-1/xyz/.svn/entries' stored as 'testsuite-out/xyz/.svn/entries' " ],
  355
+	              "File '%s/xyz/etc2/Logo.PNG' stored as 'testsuite-out/xyz/etc2/Logo.PNG' (22059 bytes" % pbucket(1), 
  356
+	              "File '%s/xyz/.svn/entries' stored as 'testsuite-out/xyz/.svn/entries' " % pbucket(1) ],
333 357
 	must_not_find_re = [ "not-deleted.*etc/logo.png" ])
334 358
 
335 359
 
@@ -338,7 +362,7 @@ def test_flushdir(label, dir_name):
338 362
 
339 363
 
340 364
 ## ====== Get multiple files
341  
-test_s3cmd("Get multiple files", ['get', 's3://s3cmd-autotest-1/xyz/etc2/Logo.PNG', 's3://s3cmd-autotest-1/xyz/etc/AtomicClockRadio.ttf', 'testsuite-out'],
  365
+test_s3cmd("Get multiple files", ['get', '%s/xyz/etc2/Logo.PNG' % pbucket(1), '%s/xyz/etc/AtomicClockRadio.ttf' % pbucket(1), 'testsuite-out'],
342 366
 	retcode = 1,
343 367
 	must_find = [ 'Destination must be a directory when downloading multiple sources.' ])
344 368
 
@@ -348,69 +372,85 @@ def test_flushdir(label, dir_name):
348 372
 
349 373
 
350 374
 ## ====== Get multiple files
351  
-test_s3cmd("Get multiple files", ['get', 's3://s3cmd-autotest-1/xyz/etc2/Logo.PNG', 's3://s3cmd-autotest-1/xyz/etc/AtomicClockRadio.ttf', 'testsuite-out'],
  375
+test_s3cmd("Get multiple files", ['get', '%s/xyz/etc2/Logo.PNG' % pbucket(1), '%s/xyz/etc/AtomicClockRadio.ttf' % pbucket(1), 'testsuite-out'],
352 376
 	must_find = [ u"saved as 'testsuite-out/Logo.PNG'", u"saved as 'testsuite-out/AtomicClockRadio.ttf'" ])
353 377
 
354 378
 ## ====== Upload files differing in capitalisation
355  
-test_s3cmd("blah.txt / Blah.txt", ['put', '-r', 'testsuite/blahBlah', 's3://s3cmd-autotest-1/'],
356  
-	must_find = [ 's3://s3cmd-autotest-1/blahBlah/Blah.txt', 's3://s3cmd-autotest-1/blahBlah/blah.txt' ])
  379
+test_s3cmd("blah.txt / Blah.txt", ['put', '-r', 'testsuite/blahBlah', pbucket(1)],
  380
+	must_find = [ '%s/blahBlah/Blah.txt' % pbucket(1), '%s/blahBlah/blah.txt' % pbucket(1)])
357 381
 
358 382
 ## ====== Copy between buckets
359  
-test_s3cmd("Copy between buckets", ['cp', 's3://s3cmd-autotest-1/xyz/etc2/Logo.PNG', 's3://s3cmd-Autotest-3/xyz/etc2/logo.png'],
360  
-	must_find = [ "File s3://s3cmd-autotest-1/xyz/etc2/Logo.PNG copied to s3://s3cmd-Autotest-3/xyz/etc2/logo.png" ])
  383
+test_s3cmd("Copy between buckets", ['cp', '%s/xyz/etc2/Logo.PNG' % pbucket(1), '%s/xyz/etc2/logo.png' % pbucket(3)],
  384
+	must_find = [ "File %s/xyz/etc2/Logo.PNG copied to %s/xyz/etc2/logo.png" % (pbucket(1), pbucket(3)) ])
361 385
 
362 386
 ## ====== Recursive copy
363  
-test_s3cmd("Recursive copy, set ACL", ['cp', '-r', '--acl-public', 's3://s3cmd-autotest-1/xyz/', 's3://s3cmd-autotest-2/copy', '--exclude', '.svn/*', '--exclude', 'too-crappy*'],
364  
-	must_find = [ "File s3://s3cmd-autotest-1/xyz/etc2/Logo.PNG copied to s3://s3cmd-autotest-2/copy/etc2/Logo.PNG",
365  
-	              "File s3://s3cmd-autotest-1/xyz/blahBlah/Blah.txt copied to s3://s3cmd-autotest-2/copy/blahBlah/Blah.txt",
366  
-	              "File s3://s3cmd-autotest-1/xyz/blahBlah/blah.txt copied to s3://s3cmd-autotest-2/copy/blahBlah/blah.txt" ],
  387
+test_s3cmd("Recursive copy, set ACL", ['cp', '-r', '--acl-public', '%s/xyz/' % pbucket(1), '%s/copy' % pbucket(2), '--exclude', '.svn/*', '--exclude', 'too-crappy*'],
  388
+	must_find = [ "File %s/xyz/etc2/Logo.PNG copied to %s/copy/etc2/Logo.PNG" % (pbucket(1), pbucket(2)),
  389
+	              "File %s/xyz/blahBlah/Blah.txt copied to %s/copy/blahBlah/Blah.txt" % (pbucket(1), pbucket(2)),
  390
+	              "File %s/xyz/blahBlah/blah.txt copied to %s/copy/blahBlah/blah.txt" % (pbucket(1), pbucket(2)) ],
367 391
 	must_not_find = [ ".svn" ])
368 392
 
  393
+## ====== Don't Put symbolic link
  394
+test_s3cmd("Don't put symbolic links", ['put', 'testsuite/etc/linked1.png', 's3://%s/xyz/' % bucket(1),],
  395
+	must_not_find_re = [ "linked1.png"])
  396
+
  397
+## ====== Put symbolic link
  398
+test_s3cmd("Put symbolic links", ['put', 'testsuite/etc/linked1.png', 's3://%s/xyz/' % bucket(1),'--follow-symlinks' ],
  399
+           must_find = [ "File 'testsuite/etc/linked1.png' stored as '%s/xyz/linked1.png'" % pbucket(1)])
  400
+
  401
+## ====== Sync symbolic links
  402
+test_s3cmd("Sync symbolic links", ['sync', 'testsuite/', 's3://%s/xyz/' % bucket(1), '--no-encrypt', '--follow-symlinks' ],
  403
+	must_find = ["File 'testsuite/etc/linked.png' stored as '%s/xyz/etc/linked.png'" % pbucket(1)],
  404
+           # Don't want to recursively copy linked directories!
  405
+           must_not_find_re = ["etc/more/linked-dir/more/give-me-more.txt",
  406
+                               "etc/brokenlink.png"],
  407
+           )
  408
+
369 409
 ## ====== Verify ACL and MIME type
370  
-test_s3cmd("Verify ACL and MIME type", ['info', 's3://s3cmd-autotest-2/copy/etc2/Logo.PNG' ],
  410
+test_s3cmd("Verify ACL and MIME type", ['info', '%s/copy/etc2/Logo.PNG' % pbucket(2) ],
371 411
 	must_find_re = [ "MIME type:.*image/png", 
372 412
 	                 "ACL:.*\*anon\*: READ",
373  
-					 "URL:.*http://s3cmd-autotest-2.s3.amazonaws.com/copy/etc2/Logo.PNG" ])
  413
+					 "URL:.*http://%s.s3.amazonaws.com/copy/etc2/Logo.PNG" % bucket(2) ])
374 414
 
375 415
 ## ====== Multi source move
376  
-test_s3cmd("Multi-source move", ['mv', '-r', 's3://s3cmd-autotest-2/copy/blahBlah/Blah.txt', 's3://s3cmd-autotest-2/copy/etc/', 's3://s3cmd-autotest-2/moved/'],
377  
-	must_find = [ "File s3://s3cmd-autotest-2/copy/blahBlah/Blah.txt moved to s3://s3cmd-autotest-2/moved/Blah.txt",
378  
-	              "File s3://s3cmd-autotest-2/copy/etc/AtomicClockRadio.ttf moved to s3://s3cmd-autotest-2/moved/AtomicClockRadio.ttf",
379  
-				  "File s3://s3cmd-autotest-2/copy/etc/TypeRa.ttf moved to s3://s3cmd-autotest-2/moved/TypeRa.ttf" ],
  416
+test_s3cmd("Multi-source move", ['mv', '-r', '%s/copy/blahBlah/Blah.txt' % pbucket(2), '%s/copy/etc/' % pbucket(2), '%s/moved/' % pbucket(2)],
  417
+	must_find = [ "File %s/copy/blahBlah/Blah.txt moved to %s/moved/Blah.txt" % (pbucket(2), pbucket(2)),
  418
+	              "File %s/copy/etc/AtomicClockRadio.ttf moved to %s/moved/AtomicClockRadio.ttf" % (pbucket(2), pbucket(2)),
  419
+				  "File %s/copy/etc/TypeRa.ttf moved to %s/moved/TypeRa.ttf" % (pbucket(2), pbucket(2)) ],
380 420
 	must_not_find = [ "blah.txt" ])
381 421
 
382 422
 ## ====== Verify move
383  
-test_s3cmd("Verify move", ['ls', '-r', 's3://s3cmd-autotest-2'],
384  
-	must_find = [ "s3://s3cmd-autotest-2/moved/Blah.txt",
385  
-	              "s3://s3cmd-autotest-2/moved/AtomicClockRadio.ttf",
386  
-				  "s3://s3cmd-autotest-2/moved/TypeRa.ttf",
387  
-				  "s3://s3cmd-autotest-2/copy/blahBlah/blah.txt" ],
388  
-	must_not_find = [ "s3://s3cmd-autotest-2/copy/blahBlah/Blah.txt",
389  
-					  "s3://s3cmd-autotest-2/copy/etc/AtomicClockRadio.ttf",
390  
-					  "s3://s3cmd-autotest-2/copy/etc/TypeRa.ttf" ])
  423
+test_s3cmd("Verify move", ['ls', '-r', pbucket(2)],
  424
+	must_find = [ "%s/moved/Blah.txt" % pbucket(2),
  425
+	              "%s/moved/AtomicClockRadio.ttf" % pbucket(2),
  426
+				  "%s/moved/TypeRa.ttf" % pbucket(2),
  427
+				  "%s/copy/blahBlah/blah.txt" % pbucket(2) ],
  428
+	must_not_find = [ "%s/copy/blahBlah/Blah.txt" % pbucket(2),
  429
+					  "%s/copy/etc/AtomicClockRadio.ttf" % pbucket(2),
  430
+					  "%s/copy/etc/TypeRa.ttf" % pbucket(2) ])
391 431
 
392 432
 ## ====== Simple delete
393  
-test_s3cmd("Simple delete", ['del', 's3://s3cmd-autotest-1/xyz/etc2/Logo.PNG'],
394  
-	must_find = [ "File s3://s3cmd-autotest-1/xyz/etc2/Logo.PNG deleted" ])
  433
+test_s3cmd("Simple delete", ['del', '%s/xyz/etc2/Logo.PNG' % pbucket(1)],
  434
+	must_find = [ "File %s/xyz/etc2/Logo.PNG deleted" % pbucket(1) ])
395 435
 
396 436
 
397 437
 ## ====== Recursive delete
398  
-test_s3cmd("Recursive delete", ['del', '--recursive', '--exclude', 'Atomic*', 's3://s3cmd-autotest-1/xyz/etc'],
399  
-	must_find = [ "File s3://s3cmd-autotest-1/xyz/etc/TypeRa.ttf deleted" ],
  438
+test_s3cmd("Recursive delete", ['del', '--recursive', '--exclude', 'Atomic*', '%s/xyz/etc' % pbucket(1)],
  439
+	must_find = [ "File %s/xyz/etc/TypeRa.ttf deleted" % pbucket(1) ],
400 440
 	must_find_re = [ "File .*\.svn/entries deleted" ],
401 441
 	must_not_find = [ "AtomicClockRadio.ttf" ])
402 442
 
403 443
 ## ====== Recursive delete all
404  
-test_s3cmd("Recursive delete all", ['del', '--recursive', '--force', 's3://s3cmd-autotest-1'],
  444
+test_s3cmd("Recursive delete all", ['del', '--recursive', '--force', pbucket(1)],
405 445
 	must_find_re = [ "File .*binary/random-crap deleted" ])
406 446
 
407 447
 
408 448
 ## ====== Remove empty bucket
409  
-test_s3cmd("Remove empty bucket", ['rb', 's3://s3cmd-autotest-1'],
410  
-	must_find = [ "Bucket 's3://s3cmd-autotest-1/' removed" ])
  449
+test_s3cmd("Remove empty bucket", ['rb', pbucket(1)],
  450
+	must_find = [ "Bucket '%s/' removed" % pbucket(1) ])
411 451
 
412 452
 
413 453
 ## ====== Remove remaining buckets
414  
-test_s3cmd("Remove remaining buckets", ['rb', '--recursive', 's3://s3cmd-autotest-2', 's3://s3cmd-Autotest-3'],
415  
-	must_find = [ "Bucket 's3://s3cmd-autotest-2/' removed",
416  
-		      "Bucket 's3://s3cmd-Autotest-3/' removed" ])
  454
+test_s3cmd("Remove remaining buckets", ['rb', '--recursive', pbucket(2), pbucket(3)],
  455
+	must_find = [ "Bucket '%s/' removed" % pbucket(2),
  456
+		      "Bucket '%s/' removed" % pbucket(3) ])
46  s3cmd
@@ -36,6 +36,41 @@ def check_args_type(args, type, verbose_type):
36 36
 		if S3Uri(arg).type != type:
37 37
 			raise ParameterError("Expecting %s instead of '%s'" % (verbose_type, arg))
38 38
 
  39
+def _fswalk_follow_symlinks(path):
  40
+        '''
  41
+        Walk filesystem, following symbolic links (but without recursion), on python2.4 and later
  42
+
  43
+        If a recursive directory link is detected, emit a warning and skip.
  44
+        '''
  45
+        assert os.path.isdir(path) # only designed for directory argument
  46
+        walkdirs = set([path])
  47
+        targets = set()
  48
+        for dirpath, dirnames, filenames in os.walk(path):
  49
+                for dirname in dirnames:
  50
+                        current = os.path.join(dirpath, dirname)
  51
+                        target = os.path.realpath(current)
  52
+                        if os.path.islink(current):
  53
+                                if target in targets:
  54
+                                        warning("Skipping recursively symlinked directory %s" % dirname)
  55
+                                else:
  56
+                                        walkdirs.add(current)
  57
+                        targets.add(target)
  58
+        for walkdir in walkdirs:
  59
+                for value in os.walk(walkdir):
  60
+                        yield value
  61
+
  62
+def fswalk(path, follow_symlinks):
  63
+        '''
  64
+        Directory tree generator
  65
+
  66
+        path (str) is the root of the directory tree to walk
  67
+
  68
+        follow_symlinks (bool) indicates whether to descend into symbolically linked directories
  69
+        '''
  70
+        if follow_symlinks:
  71
+                return _fswalk_follow_symlinks(path)
  72
+        return os.walk(path)
  73
+
39 74
 def cmd_du(args):
40 75
 	s3 = S3(Config())
41 76
 	if len(args) > 0:
@@ -620,7 +655,7 @@ def _get_filelist_local(local_uri):
620 655
 	if local_uri.isdir():
621 656
 		local_base = deunicodise(local_uri.basename())
622 657
 		local_path = deunicodise(local_uri.path())
623  
-		filelist = os.walk(local_path)
  658
+		filelist = fswalk(local_path, cfg.follow_symlinks)
624 659
 		single_file = False
625 660
 	else:
626 661
 		local_base = ""
@@ -635,9 +670,8 @@ def _get_filelist_local(local_uri):
635 670
 			if not os.path.isfile(full_name):
636 671
 				continue
637 672
 			if os.path.islink(full_name):
638  
-				## Synchronize symlinks... one day
639  
-				## for now skip over
640  
-				continue
  673
+                                if not cfg.follow_symlinks:
  674
+                                        continue
641 675
 			relative_file = unicodise(os.path.join(rel_root, f))
642 676
 			if os.path.sep != "/":
643 677
 				# Convert non-unix dir separators to '/'
@@ -1623,6 +1657,7 @@ def main():
1623 1657
 	optparser.add_option("-v", "--verbose", dest="verbosity", action="store_const", const=logging.INFO, help="Enable verbose output.")
1624 1658
 	optparser.add_option("-d", "--debug", dest="verbosity", action="store_const", const=logging.DEBUG, help="Enable debug output.")
1625 1659
 	optparser.add_option(      "--version", dest="show_version", action="store_true", help="Show s3cmd version (%s) and exit." % (PkgInfo.version))
  1660
+	optparser.add_option("-F", "--follow-symlinks", dest="follow_symlinks", action="store_true", default=False, help="Follow symbolic links as if they are regular files")
1626 1661
 
1627 1662
 	optparser.set_usage(optparser.usage + " COMMAND [parameters]")
1628 1663
 	optparser.set_description('S3cmd is a tool for managing objects in '+
@@ -1757,6 +1792,9 @@ def main():
1757 1792
 	cfg.include.extend(patterns_list)
1758 1793
 	cfg.debug_include.update(patterns_textual)
1759 1794
 
  1795
+        ## Process --follow-symlinks
  1796
+	cfg.update_option("follow_symlinks", options.follow_symlinks)
  1797
+
1760 1798
 	if cfg.encrypt and cfg.gpg_passphrase == "":
1761 1799
 		error(u"Encryption requested but no passphrase set in config file.")
1762 1800
 		error(u"Please re-run 's3cmd --configure' and supply it.")
4  s3cmd.1
@@ -138,6 +138,10 @@ Continue getting a partially downloaded file (only for \fIget\fR command). This
138 138
 \fB\-\-skip\-existing\fR
139 139
 Skip over files that exist at the destination (only for \fIget\fR and \fIsync\fR commands).
140 140
 .TP
  141
+\fB\-\-follow\-symlinks\fR
  142
+Treat local symbolic links as if they are regular files,
  143
+copying their targets to the remote. (Only for \fIput\fR and \fIsync\fR commands).
  144
+.TP
141 145
 \fB\-m\fR MIME/TYPE, \fB\-\-mime\-type\fR=MIME/TYPE
142 146
 Default MIME\-type to be set for objects stored.
143 147
 .TP
1  testsuite/etc/brokenlink.png
1  testsuite/etc/linked.png
1  testsuite/etc/linked1.png
0  testsuite/etc/more/give-me-more.txt
No changes.
1  testsuite/etc/more/linked-dir

0 notes on commit c6d507b

Please sign in to comment.
Something went wrong with that request. Please try again.