Skip to content
This repository

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse code

Add support for creating signed S3 URLs with the new signurl command

Amazon S3 supports URLs signed with an API key that permit time-limited
access to a normally private resource. Add a "signurl" command to s3cmd
that generates such URLs.

Usage: s3cmd signurl s3://bucket/object `date -d '1 year' +%s`

ie: s3cmd signurl url-to-sign expiry-in-epoch-seconds

This is a purely offline operation. Your API key and secret are not sent on
the wire. Your API key is included the generated URL, but your secret is of course
not.

No validation of the URL against S3 is performed, since this is an offline-only
operation. Use s3cmd ls or similar to test for valid objects if you need to.

The URL generated is http:// but you can simply change to https:// if you want.

For more information on signed URLs, see:

    http://s3.amazonaws.com/doc/s3-developer-guide/RESTAuthentication.html
  • Loading branch information...
commit ff6e561bbefde93cdbc84a5de0ab58f42b5bda4d 1 parent 19a529a
Craig Ringer ringerc authored

Showing 3 changed files with 98 additions and 2 deletions. Show diff stats Hide diff stats

  1. +64 2 S3/Utils.py
  2. +11 0 s3cmd
  3. +23 0 s3cmd.1
66 S3/Utils.py
@@ -13,6 +13,7 @@
13 13 import hmac
14 14 import base64
15 15 import errno
  16 +import urllib
16 17
17 18 from logging import debug, info, warning, error
18 19
@@ -319,12 +320,73 @@ def replace_nonprintables(string):
319 320 __all__.append("replace_nonprintables")
320 321
321 322 def sign_string(string_to_sign):
322   - #debug("string_to_sign: %s" % string_to_sign)
  323 + """Sign a string with the secret key, returning base64 encoded results.
  324 + By default the configured secret key is used, but may be overridden as
  325 + an argument.
  326 +
  327 + Useful for REST authentication. See http://s3.amazonaws.com/doc/s3-developer-guide/RESTAuthentication.html
  328 + """
323 329 signature = base64.encodestring(hmac.new(Config.Config().secret_key, string_to_sign, sha1).digest()).strip()
324   - #debug("signature: %s" % signature)
325 330 return signature
326 331 __all__.append("sign_string")
327 332
  333 +def sign_url(url_to_sign, expiry):
  334 + """Sign a URL in s3://bucket/object form with the given expiry
  335 + time. The object will be accessible via the signed URL until the
  336 + AWS key and secret are revoked or the expiry time is reached, even
  337 + if the object is otherwise private.
  338 +
  339 + See: http://s3.amazonaws.com/doc/s3-developer-guide/RESTAuthentication.html
  340 + """
  341 + return sign_url_base(
  342 + bucket = url_to_sign.bucket(),
  343 + object = url_to_sign.object(),
  344 + expiry = expiry
  345 + )
  346 +__all__.append("sign_url")
  347 +
  348 +def sign_url_base(**parms):
  349 + """Shared implementation of sign_url methods. Takes a hash of 'bucket', 'object' and 'expiry' as args."""
  350 + parms['expiry']=time_to_epoch(parms['expiry'])
  351 + parms['access_key']=Config.Config().access_key
  352 + debug("Expiry interpreted as epoch time %s", parms['expiry'])
  353 + signtext = 'GET\n\n\n%(expiry)d\n/%(bucket)s/%(object)s' % parms
  354 + debug("Signing plaintext: %r", signtext)
  355 + parms['sig'] = urllib.quote_plus(sign_string(signtext))
  356 + debug("Urlencoded signature: %s", parms['sig'])
  357 + return "http://%(bucket)s.s3.amazonaws.com/%(object)s?AWSAccessKeyId=%(access_key)s&Expires=%(expiry)d&Signature=%(sig)s" % parms
  358 +
  359 +def time_to_epoch(t):
  360 + """Convert time specified in a variety of forms into UNIX epoch time.
  361 + Accepts datetime.datetime, int, anything that has a strftime() method, and standard time 9-tuples
  362 + """
  363 + if isinstance(t, int):
  364 + # Already an int
  365 + return t
  366 + elif isinstance(t, tuple) or isinstance(t, time.struct_time):
  367 + # Assume it's a time 9-tuple
  368 + return int(time.mktime(t))
  369 + elif hasattr(t, 'timetuple'):
  370 + # Looks like a datetime object or compatible
  371 + return int(time.mktime(ex.timetuple()))
  372 + elif hasattr(t, 'strftime'):
  373 + # Looks like the object supports standard srftime()
  374 + return int(t.strftime('%s'))
  375 + elif isinstance(t, str) or isinstance(t, unicode):
  376 + # See if it's a string representation of an epoch
  377 + try:
  378 + return int(t)
  379 + except ValueError:
  380 + # Try to parse it as a timestamp string
  381 + try:
  382 + return time.strptime(t)
  383 + except ValueError as ex:
  384 + # Will fall through
  385 + debug("Failed to parse date with strptime: %s", ex)
  386 + pass
  387 + raise Exceptions.ParameterError('Unable to convert %r to an epoch time. Pass an epoch time. Try `date -d \'now + 1 year\' +%%s` (shell) or time.mktime (Python).' % t)
  388 +
  389 +
328 390 def check_bucket_name(bucket, dns_strict = True):
329 391 if dns_strict:
330 392 invalid = re.search("([^a-z0-9\.-])", bucket)
11 s3cmd
@@ -23,6 +23,7 @@ import locale
23 23 import subprocess
24 24 import htmlentitydefs
25 25 import socket
  26 +import S3.Exceptions
26 27
27 28 from copy import copy
28 29 from optparse import OptionParser, Option, OptionValueError, IndentedHelpFormatter
@@ -1078,6 +1079,15 @@ def cmd_sign(args):
1078 1079 signature = Utils.sign_string(string_to_sign)
1079 1080 output("Signature: %s" % signature)
1080 1081
  1082 +def cmd_signurl(args):
  1083 + expiry = args.pop()
  1084 + url_to_sign = S3Uri(args.pop())
  1085 + if url_to_sign.type != 's3':
  1086 + raise ParameterError("Must be S3Uri. Got: %s" % url_to_sign)
  1087 + debug("url to sign: %r" % url_to_sign)
  1088 + signed_url = Utils.sign_url(url_to_sign, expiry)
  1089 + output(signed_url)
  1090 +
1081 1091 def cmd_fixbucket(args):
1082 1092 def _unescape(text):
1083 1093 ##
@@ -1394,6 +1404,7 @@ def get_commands_list():
1394 1404 {"cmd":"setacl", "label":"Modify Access control list for Bucket or Files", "param":"s3://BUCKET[/OBJECT]", "func":cmd_setacl, "argc":1},
1395 1405 {"cmd":"accesslog", "label":"Enable/disable bucket access logging", "param":"s3://BUCKET", "func":cmd_accesslog, "argc":1},
1396 1406 {"cmd":"sign", "label":"Sign arbitrary string using the secret key", "param":"STRING-TO-SIGN", "func":cmd_sign, "argc":1},
  1407 + {"cmd":"signurl", "label":"Sign an S3 URL to provide limited public access with expiry", "param":"s3://BUCKET/OBJECT expiry_epoch", "func":cmd_signurl, "argc":2},
1397 1408 {"cmd":"fixbucket", "label":"Fix invalid file names in a bucket", "param":"s3://BUCKET[/PREFIX]", "func":cmd_fixbucket, "argc":1},
1398 1409
1399 1410 ## Website commands
23 s3cmd.1
@@ -63,6 +63,23 @@ Enable/disable bucket access logging
63 63 s3cmd \fBsign\fR \fISTRING-TO-SIGN\fR
64 64 Sign arbitrary string using the secret key
65 65 .TP
  66 +s3cmd \fBsignurl\fR \fIs3://BUCKET[/OBJECT]\fR \fIexpiry-in-epoch-seconds\fR
  67 +Sign an S3 URL with the secret key, producing a URL that allows access to
  68 +the named object using the credentials used to sign the URL until the date of
  69 +expiry specified in epoch-seconds has passed. This is most useful for publishing
  70 +time- or distribution-limited URLs to otherwise-private S3 objects.
  71 +.br
  72 +This is a purely offline operation. Your API key and secret are not sent on
  73 +the wire, though your public API key is included in the generated URL. Because
  74 +it's offline, no validation is done to ensure that the bucket and object actually
  75 +exist, or that this API key has permission to access them.
  76 +.br
  77 +The URL generated is http:// but you can simply change to https:// if you want.
  78 +.br
  79 +See
  80 +.B http://s3.amazonaws.com/doc/s3-developer-guide/RESTAuthentication.html
  81 +for more information on signed URLs, and the examples section below.
  82 +.TP
66 83 s3cmd \fBfixbucket\fR \fIs3://BUCKET[/PREFIX]\fR
67 84 Fix invalid file names in a bucket
68 85
@@ -421,6 +438,12 @@ about matching file names against exclude and include rules.
421 438 For example to exclude all files with ".jpg" extension except those beginning with a number use:
422 439 .PP
423 440 \-\-exclude '*.jpg' \-\-rinclude '[0-9].*\.jpg'
  441 +.PP
  442 +To produce a signed HTTP URL that allows access to the normally private s3 object
  443 +s3://mybucket/someobj (which you must have permission to access) to anybody
  444 +with the URL for one week from today, use:
  445 +.PP
  446 + s3cmd signurl s3://mybucket/someobj `date -d 'today + 1 week' +%s`
424 447 .SH SEE ALSO
425 448 For the most up to date list of options run
426 449 .B s3cmd \-\-help

0 comments on commit ff6e561

Please sign in to comment.
Something went wrong with that request. Please try again.