Permalink
Browse files

Add support for creating signed S3 URLs with the new signurl command

Amazon S3 supports URLs signed with an API key that permit time-limited
access to a normally private resource. Add a "signurl" command to s3cmd
that generates such URLs.

Usage: s3cmd signurl s3://bucket/object `date -d '1 year' +%s`

ie: s3cmd signurl url-to-sign expiry-in-epoch-seconds

This is a purely offline operation. Your API key and secret are not sent on
the wire. Your API key is included the generated URL, but your secret is of course
not.

No validation of the URL against S3 is performed, since this is an offline-only
operation. Use s3cmd ls or similar to test for valid objects if you need to.

The URL generated is http:// but you can simply change to https:// if you want.

For more information on signed URLs, see:

    http://s3.amazonaws.com/doc/s3-developer-guide/RESTAuthentication.html
  • Loading branch information...
1 parent 19a529a commit ff6e561bbefde93cdbc84a5de0ab58f42b5bda4d @ringerc ringerc committed Dec 6, 2012
Showing with 98 additions and 2 deletions.
  1. +64 −2 S3/Utils.py
  2. +11 −0 s3cmd
  3. +23 −0 s3cmd.1
View
66 S3/Utils.py
@@ -13,6 +13,7 @@
import hmac
import base64
import errno
+import urllib
from logging import debug, info, warning, error
@@ -319,12 +320,73 @@ def replace_nonprintables(string):
__all__.append("replace_nonprintables")
def sign_string(string_to_sign):
- #debug("string_to_sign: %s" % string_to_sign)
+ """Sign a string with the secret key, returning base64 encoded results.
+ By default the configured secret key is used, but may be overridden as
+ an argument.
+
+ Useful for REST authentication. See http://s3.amazonaws.com/doc/s3-developer-guide/RESTAuthentication.html
+ """
signature = base64.encodestring(hmac.new(Config.Config().secret_key, string_to_sign, sha1).digest()).strip()
- #debug("signature: %s" % signature)
return signature
__all__.append("sign_string")
+def sign_url(url_to_sign, expiry):
+ """Sign a URL in s3://bucket/object form with the given expiry
+ time. The object will be accessible via the signed URL until the
+ AWS key and secret are revoked or the expiry time is reached, even
+ if the object is otherwise private.
+
+ See: http://s3.amazonaws.com/doc/s3-developer-guide/RESTAuthentication.html
+ """
+ return sign_url_base(
+ bucket = url_to_sign.bucket(),
+ object = url_to_sign.object(),
+ expiry = expiry
+ )
+__all__.append("sign_url")
+
+def sign_url_base(**parms):
+ """Shared implementation of sign_url methods. Takes a hash of 'bucket', 'object' and 'expiry' as args."""
+ parms['expiry']=time_to_epoch(parms['expiry'])
+ parms['access_key']=Config.Config().access_key
+ debug("Expiry interpreted as epoch time %s", parms['expiry'])
+ signtext = 'GET\n\n\n%(expiry)d\n/%(bucket)s/%(object)s' % parms
+ debug("Signing plaintext: %r", signtext)
+ parms['sig'] = urllib.quote_plus(sign_string(signtext))
+ debug("Urlencoded signature: %s", parms['sig'])
+ return "http://%(bucket)s.s3.amazonaws.com/%(object)s?AWSAccessKeyId=%(access_key)s&Expires=%(expiry)d&Signature=%(sig)s" % parms
+
+def time_to_epoch(t):
+ """Convert time specified in a variety of forms into UNIX epoch time.
+ Accepts datetime.datetime, int, anything that has a strftime() method, and standard time 9-tuples
+ """
+ if isinstance(t, int):
+ # Already an int
+ return t
+ elif isinstance(t, tuple) or isinstance(t, time.struct_time):
+ # Assume it's a time 9-tuple
+ return int(time.mktime(t))
+ elif hasattr(t, 'timetuple'):
+ # Looks like a datetime object or compatible
+ return int(time.mktime(ex.timetuple()))
+ elif hasattr(t, 'strftime'):
+ # Looks like the object supports standard srftime()
+ return int(t.strftime('%s'))
+ elif isinstance(t, str) or isinstance(t, unicode):
+ # See if it's a string representation of an epoch
+ try:
+ return int(t)
+ except ValueError:
+ # Try to parse it as a timestamp string
+ try:
+ return time.strptime(t)
+ except ValueError as ex:
+ # Will fall through
+ debug("Failed to parse date with strptime: %s", ex)
+ pass
+ raise Exceptions.ParameterError('Unable to convert %r to an epoch time. Pass an epoch time. Try `date -d \'now + 1 year\' +%%s` (shell) or time.mktime (Python).' % t)
+
+
def check_bucket_name(bucket, dns_strict = True):
if dns_strict:
invalid = re.search("([^a-z0-9\.-])", bucket)
View
11 s3cmd
@@ -23,6 +23,7 @@ import locale
import subprocess
import htmlentitydefs
import socket
+import S3.Exceptions
from copy import copy
from optparse import OptionParser, Option, OptionValueError, IndentedHelpFormatter
@@ -1078,6 +1079,15 @@ def cmd_sign(args):
signature = Utils.sign_string(string_to_sign)
output("Signature: %s" % signature)
+def cmd_signurl(args):
+ expiry = args.pop()
+ url_to_sign = S3Uri(args.pop())
+ if url_to_sign.type != 's3':
+ raise ParameterError("Must be S3Uri. Got: %s" % url_to_sign)
+ debug("url to sign: %r" % url_to_sign)
+ signed_url = Utils.sign_url(url_to_sign, expiry)
+ output(signed_url)
+
def cmd_fixbucket(args):
def _unescape(text):
##
@@ -1394,6 +1404,7 @@ def get_commands_list():
{"cmd":"setacl", "label":"Modify Access control list for Bucket or Files", "param":"s3://BUCKET[/OBJECT]", "func":cmd_setacl, "argc":1},
{"cmd":"accesslog", "label":"Enable/disable bucket access logging", "param":"s3://BUCKET", "func":cmd_accesslog, "argc":1},
{"cmd":"sign", "label":"Sign arbitrary string using the secret key", "param":"STRING-TO-SIGN", "func":cmd_sign, "argc":1},
+ {"cmd":"signurl", "label":"Sign an S3 URL to provide limited public access with expiry", "param":"s3://BUCKET/OBJECT expiry_epoch", "func":cmd_signurl, "argc":2},
{"cmd":"fixbucket", "label":"Fix invalid file names in a bucket", "param":"s3://BUCKET[/PREFIX]", "func":cmd_fixbucket, "argc":1},
## Website commands
View
23 s3cmd.1
@@ -63,6 +63,23 @@ Enable/disable bucket access logging
s3cmd \fBsign\fR \fISTRING-TO-SIGN\fR
Sign arbitrary string using the secret key
.TP
+s3cmd \fBsignurl\fR \fIs3://BUCKET[/OBJECT]\fR \fIexpiry-in-epoch-seconds\fR
+Sign an S3 URL with the secret key, producing a URL that allows access to
+the named object using the credentials used to sign the URL until the date of
+expiry specified in epoch-seconds has passed. This is most useful for publishing
+time- or distribution-limited URLs to otherwise-private S3 objects.
+.br
+This is a purely offline operation. Your API key and secret are not sent on
+the wire, though your public API key is included in the generated URL. Because
+it's offline, no validation is done to ensure that the bucket and object actually
+exist, or that this API key has permission to access them.
+.br
+The URL generated is http:// but you can simply change to https:// if you want.
+.br
+See
+.B http://s3.amazonaws.com/doc/s3-developer-guide/RESTAuthentication.html
+for more information on signed URLs, and the examples section below.
+.TP
s3cmd \fBfixbucket\fR \fIs3://BUCKET[/PREFIX]\fR
Fix invalid file names in a bucket
@@ -421,6 +438,12 @@ about matching file names against exclude and include rules.
For example to exclude all files with ".jpg" extension except those beginning with a number use:
.PP
\-\-exclude '*.jpg' \-\-rinclude '[0-9].*\.jpg'
+.PP
+To produce a signed HTTP URL that allows access to the normally private s3 object
+s3://mybucket/someobj (which you must have permission to access) to anybody
+with the URL for one week from today, use:
+.PP
+ s3cmd signurl s3://mybucket/someobj `date -d 'today + 1 week' +%s`
.SH SEE ALSO
For the most up to date list of options run
.B s3cmd \-\-help

0 comments on commit ff6e561

Please sign in to comment.