diff --git a/S3/MultiPart.py b/S3/MultiPart.py index 3cb7a5f26..84eb147c9 100644 --- a/S3/MultiPart.py +++ b/S3/MultiPart.py @@ -8,7 +8,7 @@ import sys from stat import ST_SIZE from logging import debug, info, warning, error -from Utils import getTextFromXml, getTreeFromXml, formatSize, deunicodise, calculateChecksum, parseNodes, encode_to_s3 +from Utils import getTextFromXml, getTreeFromXml, formatSize, unicodise, deunicodise, calculateChecksum, parseNodes, encode_to_s3 class MultiPartUpload(object): @@ -85,26 +85,27 @@ def upload_all_parts(self): raise RuntimeError("Attempting to use a multipart upload that has not been initiated.") self.chunk_size = self.s3.config.multipart_chunk_size_mb * 1024 * 1024 + filename = unicodise(self.file.name) - if self.file.name != "": - size_left = file_size = os.stat(deunicodise(self.file.name))[ST_SIZE] + if filename != "": + size_left = file_size = os.stat(deunicodise(filename))[ST_SIZE] nr_parts = file_size / self.chunk_size + (file_size % self.chunk_size and 1) - debug("MultiPart: Uploading %s in %d parts" % (self.file.name, nr_parts)) + debug("MultiPart: Uploading %s in %d parts" % (filename, nr_parts)) else: - debug("MultiPart: Uploading from %s" % (self.file.name)) + debug("MultiPart: Uploading from %s" % filename) remote_statuses = dict() if self.s3.config.put_continue: remote_statuses = self.get_parts_information(self.uri, self.upload_id) seq = 1 - if self.file.name != "": + if filename != "": while size_left > 0: offset = self.chunk_size * (seq - 1) current_chunk_size = min(file_size - offset, self.chunk_size) size_left -= current_chunk_size labels = { - 'source' : self.file.name, + 'source' : filename, 'destination' : self.uri.uri(), 'extra' : "[part %d of %d, %s]" % (seq, nr_parts, "%d%sB" % formatSize(current_chunk_size, human_readable = True)) } @@ -112,7 +113,7 @@ def upload_all_parts(self): self.upload_part(seq, offset, current_chunk_size, labels, remote_status = remote_statuses.get(seq)) except: error(u"\nUpload of '%s' part %d failed. Use\n %s abortmp %s %s\nto abort the upload, or\n %s --upload-id %s put ...\nto continue the upload." - % (self.file.name, seq, sys.argv[0], self.uri, self.upload_id, sys.argv[0], self.upload_id)) + % (filename, seq, sys.argv[0], self.uri, self.upload_id, sys.argv[0], self.upload_id)) raise seq += 1 else: @@ -121,7 +122,7 @@ def upload_all_parts(self): offset = 0 # send from start of the buffer current_chunk_size = len(buffer) labels = { - 'source' : self.file.name, + 'source' : filename, 'destination' : self.uri.uri(), 'extra' : "[part %d, %s]" % (seq, "%d%sB" % formatSize(current_chunk_size, human_readable = True)) } @@ -131,7 +132,7 @@ def upload_all_parts(self): self.upload_part(seq, offset, current_chunk_size, labels, buffer, remote_status = remote_statuses.get(seq)) except: error(u"\nUpload of '%s' part %d failed. Use\n %s abortmp %s %s\nto abort, or\n %s --upload-id %s put ...\nto continue the upload." - % (self.file.name, seq, sys.argv[0], self.uri, self.upload_id, sys.argv[0], self.upload_id)) + % (filename, seq, sys.argv[0], self.uri, self.upload_id, sys.argv[0], self.upload_id)) raise seq += 1 diff --git a/S3/S3.py b/S3/S3.py index 9526e8f30..2adad15f3 100644 --- a/S3/S3.py +++ b/S3/S3.py @@ -591,7 +591,7 @@ def object_get(self, uri, stream, start_position = 0, extra_label = ""): if uri.type != "s3": raise ValueError("Expected URI type 's3', got '%s'" % uri.type) request = self.create_request("OBJECT_GET", uri = uri) - labels = { 'source' : uri.uri(), 'destination' : stream.name, 'extra' : extra_label } + labels = { 'source' : uri.uri(), 'destination' : unicodise(stream.name), 'extra' : extra_label } response = self.recv_file(request, stream, labels, start_position) return response @@ -1042,16 +1042,17 @@ def send_file(self, request, file, labels, buffer = '', throttle = 0, retries = S3Request.region_map[request.resource['bucket']] = region size_left = size_total = long(headers["content-length"]) + filename = unicodise(file.name) if self.config.progress_meter: progress = self.config.progress_class(labels, size_total) else: - info("Sending file '%s', please wait..." % file.name) + info("Sending file '%s', please wait..." % filename) timestamp_start = time.time() if buffer: sha256_hash = checksum_sha256_buffer(buffer, offset, size_total) else: - sha256_hash = checksum_sha256_file(file.name, offset, size_total) + sha256_hash = checksum_sha256_file(filename, offset, size_total) request.body = sha256_hash method_string, resource, headers = request.get_triplet() try: @@ -1079,7 +1080,7 @@ def send_file(self, request, file, labels, buffer = '', throttle = 0, retries = try: while (size_left > 0): - #debug("SendFile: Reading up to %d bytes from '%s' - remaining bytes: %s" % (self.config.send_chunk, file.name, size_left)) + #debug("SendFile: Reading up to %d bytes from '%s' - remaining bytes: %s" % (self.config.send_chunk, filename, size_left)) l = min(self.config.send_chunk, size_left) if buffer == '': data = file.read(l) @@ -1128,7 +1129,7 @@ def send_file(self, request, file, labels, buffer = '', throttle = 0, retries = # Connection error -> same throttle value return self.send_file(request, file, labels, buffer, throttle, retries - 1, offset, chunk_size) else: - debug("Giving up on '%s' %s" % (file.name, e)) + debug("Giving up on '%s' %s" % (filename, e)) raise S3UploadError("Upload failed for: %s" % resource['uri']) timestamp_end = time.time() @@ -1178,7 +1179,7 @@ def send_file(self, request, file, labels, buffer = '', throttle = 0, retries = time.sleep(self._fail_wait(retries)) return self.send_file(request, file, labels, buffer, throttle, retries - 1, offset, chunk_size) else: - warning("Too many failures. Giving up on '%s'" % (file.name)) + warning("Too many failures. Giving up on '%s'" % (filename)) raise S3UploadError ## Non-recoverable error @@ -1188,10 +1189,10 @@ def send_file(self, request, file, labels, buffer = '', throttle = 0, retries = if response["headers"]["etag"].strip('"\'') != md5_hash.hexdigest(): warning("MD5 Sums don't match!") if retries: - warning("Retrying upload of %s" % (file.name)) + warning("Retrying upload of %s" % (filename)) return self.send_file(request, file, labels, buffer, throttle, retries - 1, offset, chunk_size) else: - warning("Too many failures. Giving up on '%s'" % (file.name)) + warning("Too many failures. Giving up on '%s'" % (filename)) raise S3UploadError return response @@ -1209,10 +1210,11 @@ def send_file_multipart(self, file, headers, uri, size): def recv_file(self, request, stream, labels, start_position = 0, retries = _max_retries): method_string, resource, headers = request.get_triplet() + filename = unicodise(stream.name) if self.config.progress_meter: progress = self.config.progress_class(labels, 0) else: - info("Receiving file '%s', please wait..." % stream.name) + info("Receiving file '%s', please wait..." % filename) timestamp_start = time.time() try: conn = ConnMan.get(self.get_hostname(resource['bucket'])) @@ -1347,10 +1349,10 @@ def recv_file(self, request, stream, labels, start_position = 0, retries = _max_ else: # Otherwise try to compute MD5 of the output file try: - response["md5"] = hash_file_md5(stream.name) + response["md5"] = hash_file_md5(filename) except IOError, e: if e.errno != errno.ENOENT: - warning("Unable to open file: %s: %s" % (stream.name, e)) + warning("Unable to open file: %s: %s" % (filename, e)) warning("Unable to verify MD5. Assume it matches.") response["md5"] = response["headers"]["etag"]