diff --git a/examples/presigned_get_object.py b/examples/presigned_get_object.py new file mode 100644 index 000000000..e6500edc9 --- /dev/null +++ b/examples/presigned_get_object.py @@ -0,0 +1,28 @@ +# -*- coding: utf-8 -*- +# Minio Python Library for Amazon S3 Compatible Cloud Storage, (C) 2015 Minio, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import hashlib + +from minio import Minio + +__author__ = 'minio' + +# find out your s3 end point here: +# http://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region + +client = Minio('https://', + access_key='YOUR-ACCESSKEYID', + secret_key='YOUR-SECRETACCESSKEY') + +print client.presigned_get_object('mybucket', 'myobject') diff --git a/minio/minio.py b/minio/minio.py index 8b6fe2716..636fa3427 100644 --- a/minio/minio.py +++ b/minio/minio.py @@ -37,7 +37,7 @@ parse_new_multipart_upload) from .error import ResponseError from .definitions import Object -from .signer import sign_v4 +from .signer import sign_v4, presign_v4 from .xml_requests import bucket_constraint, get_complete_multipart_upload class Minio(object): @@ -329,6 +329,40 @@ def drop_all_incomplete_uploads(self, bucket): for upload in uploads: self._drop_incomplete_upload(bucket, upload.key, upload.upload_id) + def presigned_get_object(self, bucket, key, expires=None): + """ + Presigns a get object request and provides a url + """ + return self.presigned_get_partial_object(bucket, key, expires) + + def presigned_get_partial_object(self, bucket, key, expires=None, offset=0, length=0): + """ + """ + is_valid_bucket_name(bucket) + is_non_empty_string(key) + + request_range = '' + if offset is not 0 and length is not 0: + request_range = str(offset) + "-" + str(offset + length - 1) + if offset is not 0 and length is 0: + request_range = str(offset) + "-" + if offset is 0 and length is not 0: + request_range = "0-" + str(length - 1) + + method = 'GET' + url = get_target_url(self._endpoint_url, bucket=bucket, key=key) + headers = {} + + if request_range: + headers['Range'] = 'bytes=' + request_range + + method = 'GET' + presign_url = presign_v4(method=method, url=url, headers=headers, + access_key=self._access_key, + secret_key=self._secret_key) + + return presign_url + def get_object(self, bucket, key): """ Retrieves an object from a bucket. diff --git a/minio/signer.py b/minio/signer.py index 86a2f36b0..f4b1541a4 100644 --- a/minio/signer.py +++ b/minio/signer.py @@ -13,14 +13,106 @@ # See the License for the specific language governing permissions and # limitations under the License. +import collections import hashlib import hmac import binascii from datetime import datetime -from .compat import urlsplit, strtype +from .error import InvalidArgumentError +from .compat import urlsplit, strtype, urlencode from .helpers import get_region +def presign_v4(method, url, headers=None, access_key=None, secret_key=None, expires=None): + if not access_key or not secret_key: + raise InvalidArgumentError('invalid access/secret id') + + # verify only if 'None' not on expires with 0 value which should + # be an InvalidArgument is handled later below + if expires is None: + expires = 604800 + + if expires < 1 or expires > 604800: + raise InvalidArgumentError('expires param valid values are between 1 secs to 604800 secs') + + if headers is None: + headers = {} + + parsed_url = urlsplit(url) + content_hash_hex = 'UNSIGNED-PAYLOAD' + host = parsed_url.netloc + headers['host'] = host + date = datetime.utcnow() + iso8601Date = date.strftime("%Y%m%dT%H%M%SZ") + region = get_region(parsed_url.hostname) + + headers_to_sign = dict(headers) + ignored_headers = ['Authorization', 'Content-Length', 'Content-Type', + 'User-Agent'] + + for ignored_header in ignored_headers: + if ignored_header in headers_to_sign: + del headers_to_sign[ignored_header] + + query = {} + query['X-Amz-Algorithm'] = 'AWS4-HMAC-SHA256' + query['X-Amz-Credential'] = generate_credential_string(access_key, date, region) + query['X-Amz-Date'] = iso8601Date + query['X-Amz-Expires'] = expires + query['X-Amz-SignedHeaders'] = ';'.join(get_signed_headers(headers_to_sign)) + + url_components = [parsed_url.geturl()] + if query is not None: + ordered_query = collections.OrderedDict(sorted(query.items())) + query_components = [] + for component_key in ordered_query: + single_component = [component_key] + if ordered_query[component_key] is not None: + single_component.append('=') + single_component.append( + urlencode(str(ordered_query[component_key])).replace('/', '%2F')) + query_components.append(''.join(single_component)) + + query_string = '&'.join(query_components) + if query_string: + url_components.append('?') + url_components.append(query_string) + new_url = ''.join(url_components) + new_parsed_url = urlsplit(new_url) + canonical_request = generate_canonical_request(method, + new_parsed_url, + headers_to_sign, + content_hash_hex) + + canonical_request_hasher = hashlib.sha256() + canonical_request_hasher.update(canonical_request.encode('utf-8')) + canonical_request_sha256 = canonical_request_hasher.hexdigest() + + string_to_sign = generate_string_to_sign(date, region, + canonical_request_sha256) + signing_key = generate_signing_key(date, region, secret_key) + signature = hmac.new(signing_key, string_to_sign.encode('utf-8'), + hashlib.sha256).hexdigest() + + new_parsed_url = urlsplit(new_url + "&X-Amz-Signature="+signature) + return new_parsed_url.geturl() + +def get_signed_headers(headers): + headers_to_sign = dict(headers) + ignored_headers = ['Authorization', 'Content-Length', 'Content-Type', + 'User-Agent'] + + for ignored_header in ignored_headers: + if ignored_header in headers_to_sign: + del headers_to_sign[ignored_header] + + signed_headers = [] + for header in headers: + signed_headers.append(header) + signed_headers.sort() + + return signed_headers + def sign_v4(method, url, headers=None, access_key=None, secret_key=None, content_hash=None): if not access_key or not secret_key: @@ -36,7 +128,10 @@ def sign_v4(method, url, headers=None, access_key=None, secret_key=None, host = parsed_url.netloc headers['host'] = host - headers['x-amz-date'] = datetime.utcnow().strftime("%Y%m%dT%H%M%SZ") + region = get_region(parsed_url.hostname) + + date = datetime.utcnow() + headers['x-amz-date'] = date.strftime("%Y%m%dT%H%M%SZ") headers['x-amz-content-sha256'] = content_hash_hex headers_to_sign = dict(headers) @@ -75,14 +170,11 @@ def sign_v4(method, url, headers=None, access_key=None, secret_key=None, if ignored_header in headers_to_sign: del headers_to_sign[ignored_header] - canonical_request, signed_headers = generate_canonical_request(method, - parsed_url, - headers_to_sign, - content_hash_hex) - - region = get_region(parsed_url.hostname) - - date = datetime.utcnow() + signed_headers = get_signed_headers(headers_to_sign) + canonical_request = generate_canonical_request(method, + parsed_url, + headers_to_sign, + content_hash_hex) canonical_request_hasher = hashlib.sha256() canonical_request_hasher.update(canonical_request.encode('utf-8')) @@ -91,12 +183,12 @@ def sign_v4(method, url, headers=None, access_key=None, secret_key=None, string_to_sign = generate_string_to_sign(date, region, canonical_request_sha256) signing_key = generate_signing_key(date, region, secret_key) - signed_request = hmac.new(signing_key, string_to_sign.encode('utf-8'), - hashlib.sha256).hexdigest() + signature = hmac.new(signing_key, string_to_sign.encode('utf-8'), + hashlib.sha256).hexdigest() authorization_header = generate_authorization_header(access_key, date, region, signed_headers, - signed_request) + signature) headers['authorization'] = authorization_header @@ -134,21 +226,15 @@ def generate_canonical_request(method, parsed_url, headers, content_hash_hex): lines.append(';'.join(signed_headers)) lines.append(str(content_hash_hex)) - return '\n'.join(lines), signed_headers + return '\n'.join(lines) def generate_string_to_sign(date, region, request_hash): formatted_date_time = date.strftime("%Y%m%dT%H%M%SZ") - formatted_date = date.strftime("%Y%m%d") - - scope = '/'.join([formatted_date, - region, - 's3', - 'aws4_request']) return '\n'.join(['AWS4-HMAC-SHA256', formatted_date_time, - scope, + generate_scope_string(date, region), request_hash]) @@ -165,15 +251,21 @@ def generate_signing_key(date, region, secret): return hmac.new(key4, 'aws4_request'.encode('utf-8'), hashlib.sha256).digest() +def generate_scope_string(date, region): + formatted_date = date.strftime("%Y%m%d") + scope = '/'.join([formatted_date, + region, + 's3', + 'aws4_request']) + return scope + +def generate_credential_string(access_key, date, region): + return access_key + '/' + generate_scope_string(date, region) def generate_authorization_header(access_key, date, region, signed_headers, - signed_request): - formatted_date = date.strftime("%Y%m%d") + signature): signed_headers_string = ';'.join(signed_headers) - auth_header = "AWS4-HMAC-SHA256 Credential=" + access_key + "/" + \ - formatted_date + "/" + region + \ - "/s3/aws4_request, SignedHeaders=" + \ - signed_headers_string + \ - ", Signature=" + \ - signed_request + credential = generate_credential_string(access_key, date, region) + auth_header = "AWS4-HMAC-SHA256 Credential=" + credential + ", SignedHeaders=" + \ + signed_headers_string + ", Signature=" + signature return auth_header diff --git a/tests/unit/compat.py b/tests/unit/compat.py index bd88698f9..93ead7f19 100644 --- a/tests/unit/compat.py +++ b/tests/unit/compat.py @@ -16,9 +16,9 @@ import sys try: - from urllib.parse import urlparse as compat_urllib_parse + from urllib.parse import urlparse as urlsplit except ImportError: # python 2 - from urlparse import urlparse as compat_urllib_parse + from urlparse import urlparse as urlsplit strtype = None if sys.version_info < (3, 0): diff --git a/tests/unit/sign_test.py b/tests/unit/sign_test.py index 50b309b78..5e11a13fe 100644 --- a/tests/unit/sign_test.py +++ b/tests/unit/sign_test.py @@ -1,14 +1,14 @@ # -*- coding: utf-8 -*- # Minio Python Library for Amazon S3 Compatible Cloud Storage, (C) 2015 Minio, Inc. # -# Licensed under the Apache License, Version 2.0 (the "License"); +# Licensed under the Apache License, Version 2.0 (the 'License'); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, +# distributed under the License is distributed on an 'AS IS' BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. @@ -17,12 +17,14 @@ from unittest import TestCase from datetime import datetime -from nose.tools import eq_ +from nose.tools import eq_, raises import pytz as pytz -from .compat import compat_urllib_parse -from minio.signer import generate_canonical_request, generate_string_to_sign, generate_signing_key, \ - generate_authorization_header +from minio.signer import generate_canonical_request, generate_string_to_sign, \ + generate_signing_key, generate_authorization_header, presign_v4 +from minio.error import InvalidArgumentError + +from .compat import urlsplit __author__ = 'minio' @@ -31,7 +33,7 @@ class CanonicalRequestTest(TestCase): def test_simple_request(self): - url = compat_urllib_parse('http://localhost:9000/hello') + url = urlsplit('http://localhost:9000/hello') expected_signed_headers = ['x-amz-content-sha256', 'x-amz-date'] expected_request_array = ['PUT', '/hello', '', 'x-amz-content-sha256:' + @@ -40,20 +42,18 @@ def test_simple_request(self): empty_hash] expected_request = '\n'.join(expected_request_array) - - actual_request, actual_signed_headers = generate_canonical_request('PUT', - url, - {'X-Amz-Date': 'dateString', - ' x-Amz-Content-sha256\t': "\t" + - empty_hash + - " "}, - empty_hash) + actual_request = generate_canonical_request('PUT', + url, + {'X-Amz-Date': 'dateString', + ' x-Amz-Content-sha256\t': '\t' + + empty_hash + + ' '}, + empty_hash) eq_(expected_request, actual_request) - eq_(expected_signed_headers, actual_signed_headers) def test_request_with_query(self): - url = compat_urllib_parse('http://localhost:9000/hello?c=d&e=f&a=b') + url = urlsplit('http://localhost:9000/hello?c=d&e=f&a=b') expected_signed_headers = ['x-amz-content-sha256', 'x-amz-date'] expected_request_array = ['PUT', '/hello', 'a=b&c=d&e=f', 'x-amz-content-sha256:' + empty_hash, @@ -63,24 +63,24 @@ def test_request_with_query(self): expected_request = '\n'.join(expected_request_array) - actual_request, actual_signed_headers = generate_canonical_request('PUT', - url, - {'X-Amz-Date': 'dateString', - ' x-Amz-Content-sha256\t': "\t" + - empty_hash + - " "}, - empty_hash) + actual_request = generate_canonical_request('PUT', + url, + {'X-Amz-Date': 'dateString', + ' x-Amz-Content-sha256\t': '\t' + + empty_hash + + ' '}, + empty_hash) eq_(expected_request, actual_request) class StringToSignTest(TestCase): def test_signing_key(self): - expected_signing_key_list = ["AWS4-HMAC-SHA256", "20150620T010203Z", - "20150620/milkyway/s3/aws4_request", + expected_signing_key_list = ['AWS4-HMAC-SHA256', '20150620T010203Z', + '20150620/milkyway/s3/aws4_request', 'request_hash'] - actual_signing_key = generate_string_to_sign(dt, "milkyway", 'request_hash') + actual_signing_key = generate_string_to_sign(dt, 'milkyway', 'request_hash') eq_('\n'.join(expected_signing_key_list), actual_signing_key) @@ -100,9 +100,18 @@ def test_generate_signing_key(self): class AuthorizationHeaderTest(TestCase): def test_generate_authentication_header(self): - expected_authorization_header = "AWS4-HMAC-SHA256 Credential=public_key/20150620/region/s3/aws4_request, " \ - "SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=signed_request" + expected_authorization_header = 'AWS4-HMAC-SHA256 Credential=public_key/20150620/region/s3/aws4_request, ' \ + 'SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=signed_request' actual_authorization_header = generate_authorization_header('public_key', dt, 'region', ['host', 'x-amz-content-sha256', 'x-amz-date'], 'signed_request') eq_(expected_authorization_header, actual_authorization_header) + +class PresignURLTest(TestCase): + @raises(InvalidArgumentError) + def test_presigned_no_access_key(self): + presign_v4('GET', 'http://localhost:9000/hello') + + @raises(InvalidArgumentError) + def test_presigned_invalid_expires(self): + presign_v4('GET', 'http://localhost:9000/hello', headers={}, access_key='accesskey', secret_key='secretkey', expires=0)