Merge pull request #808 from robd003/master

Added support for S3 Glacier archive retrieval speed options

README.md

@@ -32,7 +32,7 @@ give these keys to S3cmd. Think of them as if they were a username and password
 
 At the time of this writing the costs of using S3 are (in USD):
 
-$0.03 per GB per month of storage space used
+$0.026 per GB per month of storage space used
 
 plus
 
@@ -43,7 +43,8 @@ plus
 $0.000 per GB - first 1GB / month data downloaded
 $0.090 per GB - up to 10 TB / month data downloaded
 $0.085 per GB - next 40 TB / month data downloaded
-$0.070 per GB - data downloaded / month over 50 TB
+$0.070 per GB - next 100 TB / month data downloaded
+$0.050 per GB - data downloaded / month over 150 TB
 
 plus
 

S3/Config.py

@@ -52,6 +52,7 @@ class Config(object):
     skip_existing = False
     recursive = False
     restore_days = 1
+    restore_priority = "Standard"
     acl_public = None
     acl_grants = []
     acl_revokes = []

S3/S3.py

@@ -734,9 +734,16 @@ def object_delete(self, uri):
     def object_restore(self, uri):
         if uri.type != "s3":
             raise ValueError("Expected URI type 's3', got '%s'" % uri.type)
-        body = '<RestoreRequest xmlns="http://s3.amazonaws.com/doc/2006-3-01">'
+        if self.config.restore_days < 1:
+            raise ParameterError("You must restore a file for 1 or more days")
+        if self.config.restore_priority not in ['Standard', 'Expedited', 'Bulk']:
+            raise ParameterError("Valid restoration priorities: bulk, standard, expedited")
+        body =   '<RestoreRequest xmlns="http://s3.amazonaws.com/doc/2006-3-01">'
         body += ('  <Days>%s</Days>' % self.config.restore_days)
-        body += '</RestoreRequest>'
+        body +=  '  <GlacierJobParameters>'
+        body += ('    <Tier>%s</Tier>' % self.config.restore_priority)
+        body +=  '  </GlacierJobParameters>'
+        body +=  '</RestoreRequest>'
         request = self.create_request("OBJECT_POST", uri = uri, extra = "?restore", body = body)
         response = self.send_request(request)
         debug("Received response '%s'" % (response))

s3cmd

@@ -732,11 +732,17 @@ def cmd_object_restore(args):
     if cfg.restore_days < 1:
         raise ParameterError("You must restore a file for 1 or more days")
 
+    # accept case-insensitive argument but fix it to match S3 API
+    if cfg.restore_priority.title() not in ['Standard', 'Expedited', 'Bulk']:
+        raise ParameterError("Valid restoration priorities: bulk, standard, expedited")
+    else:
+        cfg.restore_priority = cfg.restore_priority.title()
+
     remote_list, exclude_list, remote_total_size = fetch_remote_list(args, require_attribs = False, recursive = cfg.recursive)
 
     remote_count = len(remote_list)
 
-    info(u"Summary: Restoring %d remote files for %d days" % (remote_count, cfg.restore_days))
+    info(u"Summary: Restoring %d remote files for %d days at %s priority" % (remote_count, cfg.restore_days, cfg.restore_priority))
 
     if cfg.dry_run:
         for key in exclude_list:
@@ -2559,7 +2565,8 @@ def main():
     optparser.add_option(      "--acl-grant", dest="acl_grants", type="s3acl", action="append", metavar="PERMISSION:EMAIL or USER_CANONICAL_ID", help="Grant stated permission to a given amazon user. Permission is one of: read, write, read_acp, write_acp, full_control, all")
     optparser.add_option(      "--acl-revoke", dest="acl_revokes", type="s3acl", action="append", metavar="PERMISSION:USER_CANONICAL_ID", help="Revoke stated permission for a given amazon user. Permission is one of: read, write, read_acp, wr     ite_acp, full_control, all")
 
-    optparser.add_option("-D",  "--restore-days", dest="restore_days", action="store", help="Number of days to keep restored file available (only for 'restore' command).", metavar="NUM")
+    optparser.add_option("-D", "--restore-days", dest="restore_days", action="store", help="Number of days to keep restored file available (only for 'restore' command).", metavar="NUM")
+    optparser.add_option(      "--restore-priority", dest="restore_priority", action="store", choices=['standard', 'expedited', 'bulk'], help="Priority for restoring files from S3 Glacier (only for 'restore' command). Choices available: bulk, standard, expedited")
 
     optparser.add_option(      "--delete-removed", dest="delete_removed", action="store_true", help="Delete remote objects with no corresponding local file [sync]")
     optparser.add_option(      "--no-delete-removed", dest="delete_removed", action="store_false", help="Don't delete remote objects.")
@@ -2623,7 +2630,7 @@ def main():
     optparser.add_option(      "--no-progress", dest="progress_meter", action="store_false", help="Don't display progress meter (default on non-TTY).")
     optparser.add_option(      "--stats", dest="stats", action="store_true", help="Give some file-transfer stats.")
     optparser.add_option(      "--enable", dest="enable", action="store_true", help="Enable given CloudFront distribution (only for [cfmodify] command)")
-    optparser.add_option(      "--disable", dest="enable", action="store_false", help="Enable given CloudFront distribution (only for [cfmodify] command)")
+    optparser.add_option(      "--disable", dest="enable", action="store_false", help="Disable given CloudFront distribution (only for [cfmodify] command)")
     optparser.add_option(      "--cf-invalidate", dest="invalidate_on_cf", action="store_true", help="Invalidate the uploaded filed in CloudFront. Also see [cfinval] command.")
     # joseprio: adding options to invalidate the default index and the default
     # index root

s3cmd.1

@@ -112,6 +112,9 @@ Set or delete expiration rule for the bucket
 s3cmd \fBsetlifecycle\fR \fIFILE s3://BUCKET\fR
 Upload a lifecycle policy for the bucket
 .TP
+s3cmd \fBgetlifecycle\fR \fIs3://BUCKET\fR
+Get a lifecycle policy for the bucket
+.TP
 s3cmd \fBdellifecycle\fR \fIs3://BUCKET\fR
 Remove a lifecycle policy for the bucket
 
@@ -182,6 +185,9 @@ AWS Access Key
 \fB\-\-secret_key\fR=SECRET_KEY
 AWS Secret Key
 .TP
+\fB\-\-access_token\fR=ACCESS_TOKEN
+AWS Access Token
+.TP
 \fB\-n\fR, \fB\-\-dry\-run\fR
 Only show what should be uploaded or downloaded but
 don't actually do it. May still perform S3 requests to
@@ -260,6 +266,31 @@ ite_acp, full_control, all
 Number of days to keep restored file available (only
 for 'restore' command).
 .TP
+\fB\-\-restore\-priority\fR=bulk,standard,expedited
+You can specify one of the following when restoring an
+S3 Glacier archived object (only for 'restore' command):
+.TP
+Expedited - Expedited retrievals allow you to quickly
+access your data when occasional urgent requests for a
+subset of archives are required. For all but the largest
+archived objects (250 MB+), data accessed using Expedited
+retrievals are typically made available within 1–5 minutes.
+There are two types of Expedited retrievals: On-Demand and
+Provisioned. On-Demand requests are similar to EC2 On-Demand
+instances and are available most of the time. Provisioned
+requests are guaranteed to be available when you need them.
+.TP
+Standard - Standard retrievals allow you to access any of
+your archived objects within several hours. Standard
+retrievals typically complete within 3–5 hours. This is the
+default option for retrieval requests that do not specify
+the retrieval option.
+.TP
+Bulk - Bulk retrievals are Amazon Glacier’s lowest-cost
+retrieval option, enabling you to retrieve large amounts,
+even petabytes, of data inexpensively in a day. Bulk
+retrievals typically complete within 5–12 hours.
+.TP
 \fB\-\-delete\-removed\fR
 Delete remote objects with no corresponding local file
 [sync]
@@ -277,6 +308,10 @@ Perform deletes after new uploads [sync]
 \fB\-\-max\-delete\fR=NUM
 Do not delete more than NUM files. [del] and [sync]
 .TP
+\fB\-\-limit\fR=NUM
+Limit number of objects returned in the response body
+(only for [ls] and [la] commands)
+.TP
 \fB\-\-add\-destination\fR=ADDITIONAL_DESTINATIONS
 Additional destination for parallel uploads, in
 addition to last arg.  May be repeated.