Skip to content
Browse files

2009-05-27 Michal Ludvig <michal@logix.cz>

	* S3/SortedDict.py: Add case-sensitive mode.
	* s3cmd, S3/S3.py, S3/Config.py: Use SortedDict() in 
	  case-sensitive mode to avoid dropping filenames
	  differing only in capitalisation
	* run-tests.py: Testsuite for the above.
	* NEWS: Updated.



git-svn-id: https://s3tools.svn.sourceforge.net/svnroot/s3tools/s3cmd/trunk@386 830e0280-6d2a-0410-9c65-932aecc39d9d
  • Loading branch information...
1 parent 17901aa commit dc1c96cf069f076d18964f352ddfae00c3e99ab8 @mludvig mludvig committed May 27, 2009
Showing with 43 additions and 23 deletions.
  1. +9 −0 ChangeLog
  2. +2 −0 NEWS
  3. +1 −1 S3/Config.py
  4. +4 −4 S3/S3.py
  5. +18 −12 S3/SortedDict.py
  6. +3 −0 run-tests.py
  7. +6 −6 s3cmd
View
9 ChangeLog
@@ -1,3 +1,12 @@
+2009-05-27 Michal Ludvig <michal@logix.cz>
+
+ * S3/SortedDict.py: Add case-sensitive mode.
+ * s3cmd, S3/S3.py, S3/Config.py: Use SortedDict() in
+ case-sensitive mode to avoid dropping filenames
+ differing only in capitalisation
+ * run-tests.py: Testsuite for the above.
+ * NEWS: Updated.
+
2009-03-20 Michal Ludvig <michal@logix.cz>
* S3/S3.py: Re-sign requests before retrial to avoid
View
2 NEWS
@@ -2,6 +2,8 @@ s3cmd 1.0.0
===========
* New command 'sign' for signing for instance
the POST upload policies.
+* Fixed handling of filenames that differ only in
+ capitalisation (eg blah.txt vs Blah.TXT).
s3cmd 0.9.9 - 2009-02-17
===========
View
2 S3/Config.py
@@ -27,7 +27,7 @@ class Config(object):
recv_chunk = 4096
list_md5 = False
human_readable_sizes = False
- extra_headers = SortedDict()
+ extra_headers = SortedDict(ignore_case = True)
force = False
get_continue = False
skip_existing = False
View
8 S3/S3.py
@@ -27,7 +27,7 @@
class S3Request(object):
def __init__(self, s3, method_string, resource, headers, params = {}):
self.s3 = s3
- self.headers = SortedDict(headers or {})
+ self.headers = SortedDict(headers or {}, ignore_case = True)
self.resource = resource
self.method_string = method_string
self.params = params
@@ -195,7 +195,7 @@ def _get_common_prefixes(data):
return response
def bucket_create(self, bucket, bucket_location = None):
- headers = SortedDict()
+ headers = SortedDict(ignore_case = True)
body = ""
if bucket_location and bucket_location.strip().upper() != "US":
body = "<CreateBucketConfiguration><LocationConstraint>"
@@ -235,7 +235,7 @@ def object_put(self, filename, uri, extra_headers = None, extra_label = ""):
size = os.stat(filename)[ST_SIZE]
except IOError, e:
raise InvalidFileError(u"%s: %s" % (unicodise(filename), e.strerror))
- headers = SortedDict()
+ headers = SortedDict(ignore_case = True)
if extra_headers:
headers.update(extra_headers)
headers["content-length"] = size
@@ -273,7 +273,7 @@ def object_copy(self, src_uri, dst_uri, extra_headers = None):
raise ValueError("Expected URI type 's3', got '%s'" % src_uri.type)
if dst_uri.type != "s3":
raise ValueError("Expected URI type 's3', got '%s'" % dst_uri.type)
- headers = SortedDict()
+ headers = SortedDict(ignore_case = True)
headers['x-amz-copy-source'] = "/%s/%s" % (src_uri.bucket(), self.urlencode_string(src_uri.object()))
if self.config.acl_public:
headers["x-amz-acl"] = "public-read"
View
30 S3/SortedDict.py
@@ -17,11 +17,19 @@ def next(self):
raise StopIteration
class SortedDict(dict):
- keys_sort_lowercase = True
+ def __init__(self, mapping = {}, ignore_case = True, **kwargs):
+ """
+ WARNING: SortedDict() with ignore_case==True will
+ drop entries differing only in capitalisation!
+ Eg: SortedDict({'auckland':1, 'Auckland':2}).keys() => ['Auckland']
+ With ignore_case==False it's all right
+ """
+ dict.__init__(self, mapping, **kwargs)
+ self.ignore_case = ignore_case
def keys(self):
keys = dict.keys(self)
- if self.keys_sort_lowercase:
+ if self.ignore_case:
# Translation map
xlat_map = BidirMap()
for key in keys:
@@ -38,18 +46,16 @@ def __iter__(self):
return SortedDictIterator(self, self.keys())
if __name__ == "__main__":
- d = SortedDict()
- d['AWS'] = 1
- d['Action'] = 2
- d['america'] = 3
- d.keys_sort_lowercase = True
- print "Wanted: Action, america, AWS,"
+ d = { 'AWS' : 1, 'Action' : 2, 'america' : 3, 'Auckland' : 4, 'America' : 5 }
+ sd = SortedDict(d)
+ print "Wanted: Action, america, Auckland, AWS, [ignore case]"
print "Got: ",
- for key in d:
+ for key in sd:
print "%s," % key,
- print " __iter__()"
- d.keys_return_lowercase = True
+ print " [used: __iter__()]"
+ d = SortedDict(d, ignore_case = False)
+ print "Wanted: AWS, Action, Auckland, america, [case sensitive]"
print "Got: ",
for key in d.keys():
print "%s," % key,
- print " keys()"
+ print " [used: keys()]"
View
3 run-tests.py
@@ -338,6 +338,9 @@ def test_flushdir(label, dir_name):
test_s3cmd("Copy between buckets", ['cp', 's3://s3cmd-autotest-1/xyz/etc2/Logo.PNG', 's3://s3cmd-Autotest-3'],
must_find = [ "File s3://s3cmd-autotest-1/xyz/etc2/Logo.PNG copied to s3://s3cmd-Autotest-3/xyz/etc2/Logo.PNG" ])
+## ====== Upload files differing in capitalisation
+test_s3cmd("blah.txt / Blah.txt", ['put', '-r', 'testsuite/blahBlah', 's3://s3cmd-autotest-1/'],
+ must_find = [ 's3://s3cmd-autotest-1/blahBlah/Blah.txt', 's3://s3cmd-autotest-1/blahBlah/blah.txt' ])
## ====== Simple delete
test_s3cmd("Simple delete", ['del', 's3://s3cmd-autotest-1/xyz/etc2/Logo.PNG'],
View
12 s3cmd
@@ -187,7 +187,7 @@ def cmd_bucket_delete(args):
def fetch_local_list(args, recursive = None):
local_uris = []
- local_list = SortedDict()
+ local_list = SortedDict(ignore_case = False)
single_file = False
if type(args) not in (list, tuple):
@@ -220,7 +220,7 @@ def fetch_local_list(args, recursive = None):
def fetch_remote_list(args, require_attribs = False, recursive = None):
remote_uris = []
- remote_list = SortedDict()
+ remote_list = SortedDict(ignore_case = False)
if type(args) not in (list, tuple):
args = [args]
@@ -586,7 +586,7 @@ def _get_filelist_local(local_uri):
local_path = deunicodise(local_uri.dirname())
filelist = [( local_path, [], [deunicodise(local_uri.basename())] )]
single_file = True
- loc_list = SortedDict()
+ loc_list = SortedDict(ignore_case = False)
for root, dirs, files in filelist:
rel_root = root.replace(local_path, local_base, 1)
for f in files:
@@ -637,7 +637,7 @@ def _get_filelist_remote(remote_uri, recursive = True):
rem_base = rem_base[:rem_base.rfind('/')+1]
remote_uri = S3Uri("s3://%s/%s" % (remote_uri.bucket(), rem_base))
rem_base_len = len(rem_base)
- rem_list = SortedDict()
+ rem_list = SortedDict(ignore_case = False)
break_now = False
for object in response['list']:
if object['Key'] == rem_base_original and object['Key'][-1] != os.path.sep:
@@ -664,7 +664,7 @@ def _get_filelist_remote(remote_uri, recursive = True):
def _filelist_filter_exclude_include(src_list):
info(u"Applying --exclude/--include")
cfg = Config()
- exclude_list = SortedDict()
+ exclude_list = SortedDict(ignore_case = False)
for file in src_list.keys():
debug(u"CHECK: %s" % file)
excluded = False
@@ -693,7 +693,7 @@ def _filelist_filter_exclude_include(src_list):
def _compare_filelists(src_list, dst_list, src_is_local_and_dst_is_remote):
info(u"Verifying attributes...")
cfg = Config()
- exists_list = SortedDict()
+ exists_list = SortedDict(ignore_case = False)
for file in src_list.keys():
debug(u"CHECK: %s" % file)

0 comments on commit dc1c96c

Please sign in to comment.
Something went wrong with that request. Please try again.