Skip to content
This repository

Add --delete-after for sync command #30

Merged
merged 3 commits into from about 1 year ago

2 participants

Matt Domsch Michal Ludvig
Matt Domsch
Collaborator

Allow user to specify --delete-after option, used by sync command to delay deleting removed files until after new files have been uploaded. This is necessary for proper operation of a yum repository (and many other models) during the interim period when a sync has started but not all the metadata has yet been synced out. In this case, the still-valid repodata/ directory files get deleted before the new content (packages and new repodata/ files) get uploaded, causing the mirror to be useless in the meantime, which could be many hours.

Michal Ludvig mludvig merged commit b62ce58 into from February 18, 2013
Michal Ludvig mludvig closed this February 18, 2013
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
This page is out of date. Refresh to see the latest.
1  S3/Config.py
@@ -53,6 +53,7 @@ class Config(object):
53 53
         #'acl',     # Full ACL (not yet supported)
54 54
     ]
55 55
     delete_removed = False
  56
+    delete_after = False
56 57
     _doc['delete_removed'] = "[sync] Remove remote S3 objects when local file has been deleted"
57 58
     gpg_passphrase = ""
58 59
     gpg_command = ""
57  s3cmd
@@ -582,6 +582,17 @@ def cmd_info(args):
582 582
                 raise
583 583
 
584 584
 def cmd_sync_remote2remote(args):
  585
+    def _do_deletes(s3, dst_list):
  586
+        # Delete items in destination that are not in source
  587
+        if cfg.dry_run:
  588
+            for key in dst_list:
  589
+                output(u"delete: %s" % dst_list[key]['object_uri_str'])
  590
+        else:
  591
+            for key in dst_list:
  592
+                uri = S3Uri(dst_list[key]['object_uri_str'])
  593
+                s3.object_delete(uri)
  594
+                output(u"deleted: '%s'" % uri)
  595
+
585 596
     s3 = S3(Config())
586 597
 
587 598
     # Normalise s3://uri (e.g. assert trailing slash)
@@ -621,15 +632,8 @@ def cmd_sync_remote2remote(args):
621 632
         return
622 633
 
623 634
     # Delete items in destination that are not in source
624  
-    if cfg.delete_removed:
625  
-        if cfg.dry_run:
626  
-            for key in dst_list:
627  
-                output(u"delete: %s" % dst_list[key]['object_uri_str'])
628  
-        else:
629  
-            for key in dst_list:
630  
-                uri = S3Uri(dst_list[key]['object_uri_str'])
631  
-                s3.object_delete(uri)
632  
-                output(u"deleted: '%s'" % uri)
  635
+    if cfg.delete_removed and not cfg.delete_after:
  636
+        _do_deletes(s3, dst_list)
633 637
 
634 638
     # Perform the synchronization of files
635 639
     timestamp_start = time.time()
@@ -655,6 +659,10 @@ def cmd_sync_remote2remote(args):
655 659
     else:
656 660
         info(outstr)
657 661
 
  662
+    # Delete items in destination that are not in source
  663
+    if cfg.delete_removed and cfg.delete_after:
  664
+        _do_deletes(s3, dst_list)
  665
+
658 666
 def cmd_sync_remote2local(args):
659 667
     def _parse_attrs_header(attrs_header):
660 668
         attrs = {}
@@ -663,6 +671,11 @@ def cmd_sync_remote2local(args):
663 671
             attrs[key] = val
664 672
         return attrs
665 673
 
  674
+    def _do_deletes(local_list):
  675
+        for key in local_list:
  676
+            os.unlink(local_list[key]['full_name'])
  677
+            output(u"deleted: %s" % local_list[key]['full_name_unicode'])
  678
+
666 679
     s3 = S3(Config())
667 680
 
668 681
     destination_base = args[-1]
@@ -709,10 +722,8 @@ def cmd_sync_remote2local(args):
709 722
         warning(u"Exitting now because of --dry-run")
710 723
         return
711 724
 
712  
-    if cfg.delete_removed:
713  
-        for key in local_list:
714  
-            os.unlink(local_list[key]['full_name'])
715  
-            output(u"deleted: %s" % local_list[key]['full_name_unicode'])
  725
+    if cfg.delete_removed and not cfg.delete_after:
  726
+        _do_deletes(local_list)
716 727
 
717 728
     total_size = 0
718 729
     total_elapsed = 0.0
@@ -804,6 +815,9 @@ def cmd_sync_remote2local(args):
804 815
     else:
805 816
         info(outstr)
806 817
 
  818
+    if cfg.delete_removed and cfg.delete_after:
  819
+        _do_deletes(local_list)
  820
+
807 821
 def cmd_sync_local2remote(args):
808 822
     def _build_attr_header(src):
809 823
         import pwd, grp
@@ -835,6 +849,12 @@ def cmd_sync_local2remote(args):
835 849
         for k in attrs: result += "%s:%s/" % (k, attrs[k])
836 850
         return { 'x-amz-meta-s3cmd-attrs' : result[:-1] }
837 851
 
  852
+    def _do_deletes(s3, remote_list):
  853
+        for key in remote_list:
  854
+            uri = S3Uri(remote_list[key]['object_uri_str'])
  855
+            s3.object_delete(uri)
  856
+            output(u"deleted: '%s'" % uri)
  857
+
838 858
     s3 = S3(cfg)
839 859
 
840 860
     if cfg.encrypt:
@@ -894,11 +914,8 @@ def cmd_sync_local2remote(args):
894 914
         warning(u"Exitting now because of --dry-run")
895 915
         return
896 916
 
897  
-    if cfg.delete_removed:
898  
-        for key in remote_list:
899  
-            uri = S3Uri(remote_list[key]['object_uri_str'])
900  
-            s3.object_delete(uri)
901  
-            output(u"deleted: '%s'" % uri)
  917
+    if cfg.delete_removed and not cfg.delete_after:
  918
+        _do_deletes(s3, remote_list)
902 919
 
903 920
     uploaded_objects_list = []
904 921
     total_size = 0
@@ -946,6 +963,9 @@ def cmd_sync_local2remote(args):
946 963
     else:
947 964
         info(outstr)
948 965
 
  966
+    if cfg.delete_removed and cfg.delete_after:
  967
+        _do_deletes(s3, remote_list)
  968
+
949 969
     if cfg.invalidate_on_cf:
950 970
         if len(uploaded_objects_list) == 0:
951 971
             info("Nothing to invalidate in CloudFront")
@@ -1499,6 +1519,7 @@ def main():
1499 1519
 
1500 1520
     optparser.add_option(      "--delete-removed", dest="delete_removed", action="store_true", help="Delete remote objects with no corresponding local file [sync]")
1501 1521
     optparser.add_option(      "--no-delete-removed", dest="delete_removed", action="store_false", help="Don't delete remote objects.")
  1522
+    optparser.add_option(      "--delete-after", dest="delete_after", action="store_true", help="Perform deletes after new uploads [sync]")
1502 1523
     optparser.add_option("-p", "--preserve", dest="preserve_attrs", action="store_true", help="Preserve filesystem attributes (mode, ownership, timestamps). Default for [sync] command.")
1503 1524
     optparser.add_option(      "--no-preserve", dest="preserve_attrs", action="store_false", help="Don't store FS attributes")
1504 1525
     optparser.add_option(      "--exclude", dest="exclude", action="append", metavar="GLOB", help="Filenames and paths matching GLOB will be excluded from sync")
3  s3cmd.1
@@ -186,6 +186,9 @@ Delete remote objects with no corresponding local file
186 186
 \fB\-\-no\-delete\-removed\fR
187 187
 Don't delete remote objects.
188 188
 .TP
  189
+\fB\-\-delete\-after\fR
  190
+Perform deletes after new uploads [sync].
  191
+.TP
189 192
 \fB\-p\fR, \fB\-\-preserve\fR
190 193
 Preserve filesystem attributes (mode, ownership,
191 194
 timestamps). Default for [sync] command.
Commit_comment_tip

Tip: You can add notes to lines in a file. Hover to the left of a line to make a note

Something went wrong with that request. Please try again.