Permalink
Browse files

Allow precomputation of the file md5 sums.

  • Loading branch information...
1 parent 23d133a commit cc605705eca3eaf41004da209f9c086db933073c Ryan Lim committed Feb 27, 2012
Showing with 26 additions and 2 deletions.
  1. +1 −0 S3/Config.py
  2. +20 −2 S3/FileLists.py
  3. +5 −0 s3cmd
View
@@ -78,6 +78,7 @@ class Config(object):
log_target_prefix = ""
reduced_redundancy = False
follow_symlinks = False
+ precomputed_md5_file = ""
socket_timeout = 300
invalidate_on_cf = False
website_index = "index.html"
View
@@ -273,6 +273,17 @@ def __direction_str(is_remote):
cfg = Config()
exists_list = SortedDict(ignore_case = False)
+
+ # load precomputed md5 table
+ md5_table = {}
+ if cfg.precomputed_md5_file != "":
+ f = open(cfg.precomputed_md5_file, 'r')
+ for l in f.readlines():
+ (md5sum, name) = l.strip().split(' ', 1)
+ name = name.strip()
+ md5_table[name] = md5sum
+
+
debug("Comparing filelists (direction: %s -> %s)" % (__direction_str(src_remote), __direction_str(dst_remote)))
debug("src_list.keys: %s" % src_list.keys())
debug("dst_list.keys: %s" % dst_list.keys())
@@ -305,11 +316,18 @@ def __direction_str(is_remote):
if attribs_match and compare_md5:
try:
if src_remote == False and dst_remote == True:
- src_md5 = hash_file_md5(src_list[file]['full_name'])
+ if src_list[file]['full_name'] in md5_table:
+ src_md5 = md5_table[src_list[file]['full_name']]
+ else:
+ src_md5 = hash_file_md5(src_list[file]['full_name'])
+
dst_md5 = dst_list[file]['md5']
elif src_remote == True and dst_remote == False:
src_md5 = src_list[file]['md5']
- dst_md5 = hash_file_md5(dst_list[file]['full_name'])
+ if dst_list[file]['full_name'] in md5_table:
+ dst_md5 = md5_table[dst_list[file]['full_name']]
+ else:
+ dst_md5 = hash_file_md5(dst_list[file]['full_name'])
elif src_remote == True and dst_remote == True:
src_md5 = src_list[file]['md5']
dst_md5 = dst_list[file]['md5']
View
@@ -1548,6 +1548,7 @@ def main():
optparser.add_option("-d", "--debug", dest="verbosity", action="store_const", const=logging.DEBUG, help="Enable debug output.")
optparser.add_option( "--version", dest="show_version", action="store_true", help="Show s3cmd version (%s) and exit." % (PkgInfo.version))
optparser.add_option("-F", "--follow-symlinks", dest="follow_symlinks", action="store_true", default=False, help="Follow symbolic links as if they are regular files")
+ optparser.add_option("--precomputed-md5", dest="precompute_md5_file", action="store", help="Precomputed MD5 file")
optparser.set_usage(optparser.usage + " COMMAND [parameters]")
optparser.set_description('S3cmd is a tool for managing objects in '+
@@ -1652,6 +1653,10 @@ def main():
cfg.update_option("enable", options.enable)
cfg.update_option("acl_public", options.acl_public)
+
+ cfg.update_option("precomputed_md5_file", options.precompute_md5_file)
+
+
## Check multipart chunk constraints
if cfg.multipart_chunk_size_mb < MultiPartUpload.MIN_CHUNK_SIZE_MB:
raise ParameterError("Chunk size %d MB is too small, must be >= %d MB. Please adjust --multipart-chunk-size-mb" % (cfg.multipart_chunk_size_mb, MultiPartUpload.MIN_CHUNK_SIZE_MB))

0 comments on commit cc60570

Please sign in to comment.