Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

* S3/Utils.py: New function replace_nonprintables()

* s3cmd: Filter local filenames through the above function
  to avoid problems with uploaded filenames containing invalid 
  XML entities, eg  
* S3/S3.py: Warn if a non-printables char is passed to
  urlencode_string() - they should have been replaced earlier 
  in the processing.
* run-tests.py, TODO, NEWS: Updated.
* testsuite/crappy-file-name.tar.gz: Tarball with a crappy-named
  file. Untar for the testsuite.



git-svn-id: https://s3tools.svn.sourceforge.net/svnroot/s3tools/s3cmd/trunk@394 830e0280-6d2a-0410-9c65-932aecc39d9d
  • Loading branch information...
commit b40dd815c84befeba45538a0e97985b4b6b62615 1 parent 3be13c2
@mludvig mludvig authored
View
13 ChangeLog
@@ -1,5 +1,18 @@
2009-05-29 Michal Ludvig <michal@logix.cz>
+ * S3/Utils.py: New function replace_nonprintables()
+ * s3cmd: Filter local filenames through the above function
+ to avoid problems with uploaded filenames containing invalid
+ XML entities, eg &#08;
+ * S3/S3.py: Warn if a non-printables char is passed to
+ urlencode_string() - they should have been replaced earlier
+ in the processing.
+ * run-tests.py, TODO, NEWS: Updated.
+ * testsuite/crappy-file-name.tar.gz: Tarball with a crappy-named
+ file. Untar for the testsuite.
+
+2009-05-29 Michal Ludvig <michal@logix.cz>
+
* testsuite/blahBlah/*: Added files needed for run-tests.py
2009-05-28 Michal Ludvig <michal@logix.cz>
View
2  NEWS
@@ -8,6 +8,8 @@ s3cmd 1.0.0
* Added --recursive support for [cp] and [mv], including
multiple-source arguments, --include/--exclude, --dry-run, etc.
* Added --exclude/--include and --dry-run for [del], [setacl].
+* Neutralise characters that are invalid in XML to avoid ExpatErrors.
+ http://boodebr.org/main/python/all-about-python-and-unicode
s3cmd 0.9.9 - 2009-02-17
===========
View
12 S3/S3.py
@@ -344,19 +344,19 @@ def urlencode_string(self, string):
# systems.
# [hope that sounds reassuring ;-)]
o = ord(c)
- if (o <= 32 or # Space and below
+ if (o < 0x20 or o == 0x7f):
+ error(u"Non-printable character 0x%02x in: %s" % (o, string))
+ error(u"Please report it to s3tools-bugs@lists.sourceforge.net")
+ encoded += replace_nonprintables(c)
+ elif (o == 0x20 or # Space and below
o == 0x22 or # "
o == 0x23 or # #
- o == 0x25 or # %
+ o == 0x25 or # % (escape character)
o == 0x26 or # &
o == 0x2B or # + (or it would become <space>)
o == 0x3C or # <
o == 0x3E or # >
o == 0x3F or # ?
- o == 0x5B or # [
- o == 0x5C or # \
- o == 0x5D or # ]
- o == 0x5E or # ^
o == 0x60 or # `
o >= 123): # { and above, including >= 128 for UTF-8
encoded += "%%%02X" % o
View
23 S3/Utils.py
@@ -256,6 +256,29 @@ def unicodise_safe(string, encoding = None):
return unicodise(deunicodise(string, encoding), encoding).replace(u'\ufffd', '?')
+def replace_nonprintables(string):
+ """
+ replace_nonprintables(string)
+
+ Replaces all non-printable characters 'ch' in 'string'
+ where ord(ch) <= 26 with ^@, ^A, ... ^Z
+ """
+ new_string = ""
+ modified = 0
+ for c in string:
+ o = ord(c)
+ if (o <= 31):
+ new_string += "^" + chr(ord('@') + o)
+ modified += 1
+ elif (o == 127):
+ new_string += "^?"
+ modified += 1
+ else:
+ new_string += c
+ if modified:
+ warning("%d non-printable characters replaced in: %s" % (modified, new_string))
+ return new_string
+
def sign_string(string_to_sign):
#debug("string_to_sign: %s" % string_to_sign)
signature = base64.encodestring(hmac.new(Config.Config().secret_key, string_to_sign, sha1).digest()).strip()
View
2  TODO
@@ -12,8 +12,6 @@ TODO list for s3cmd project
- Option --mime-type should set mime type with 'cp' and 'mv'.
If possible --guess-mime-type should do as well.
- Skip files that disapper during upload. Now it fails.
- - Neutralise characters that are invalid in XML to avoid ExpatErrors.
- http://boodebr.org/main/python/all-about-python-and-unicode
- For 1.0.0
- Add 'geturl' command, both Unicode and urlencoded output.
View
2  run-tests.py
@@ -223,6 +223,8 @@ def test_flushdir(label, dir_name):
## ====== Sync to S3
test_s3cmd("Sync to S3", ['sync', 'testsuite/', 's3://s3cmd-autotest-1/xyz/', '--exclude', '.svn/*', '--exclude', '*.png', '--no-encrypt', '--exclude-from', 'testsuite/exclude.encodings' ],
+ must_find = [ "WARNING: 32 non-printable characters replaced in: crappy-file-name/non-printables ^A^B^C^D^E^F^G^H^I^J^K^L^M^N^O^P^Q^R^S^T^U^V^W^X^Y^Z^[^\^]^^^_^? +-[\]^<>%%\"'#{}`&?.end",
+ "stored as 's3://s3cmd-autotest-1/xyz/crappy-file-name/non-printables ^A^B^C^D^E^F^G^H^I^J^K^L^M^N^O^P^Q^R^S^T^U^V^W^X^Y^Z^[^\^]^^^_^? +-[\\]^<>%%\"'#{}`&?.end'" ],
must_not_find_re = [ "\.svn/", "\.png$" ])
if have_encoding:
View
2  s3cmd
@@ -638,6 +638,8 @@ def _get_filelist_local(local_uri):
## for now skip over
continue
relative_file = unicodise(os.path.join(rel_root, f))
+ if not cfg.verbatim:
+ relative_file = replace_nonprintables(relative_file)
if relative_file.startswith('./'):
relative_file = relative_file[2:]
sr = os.stat_result(os.lstat(full_name))
View
BIN  testsuite/crappy-file-name.tar.gz
Binary file not shown
Please sign in to comment.
Something went wrong with that request. Please try again.