diff --git a/examples/assume_role.py b/examples/assume_role.py new file mode 100644 index 000000000..8d95cb38a --- /dev/null +++ b/examples/assume_role.py @@ -0,0 +1,71 @@ +# -*- coding: utf-8 -*- +# MinIO Python Library for Amazon S3 Compatible Cloud Storage, (C) 2020 MinIO, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# +# AssumeRoleProvider will call the Simple Token Service (STS) to retrieve +# temporary credentials. +# +# - You can't call AssumeRole as a root user on either MinIO or AWS. +# For MinIO add a non-root user using the minio client `mc`: +# +# mc admin user add myminio YOUR-ACCESSKEYID YOUR-SECRETACCESSKEY +# On AWS you will need an IAM user with the sts:AssumeRole action allowed, +# and a target role. +# - The credentials will be valid for between 15 minutes and 12 hours. +# - An access policy can be applied to the temporary credentials. The +# resulting permissions are the intersection of the role's existing policy +# and the optionally provided policy. You cannot grant more permissions than +# those allowed by the policy of the role that is being assumed. +# - YOUR-ACCESSKEYID and YOUR-SECRETACCESSKEY are +# dummy values, please replace them with original values. +# - To use minio with AWS, the `Minio` client that is passed to the +# AssumeRoleProvider must have the endpoint 'sts.amazonaws.com', and the +# RoleARN argument must be provided. + +from minio import Minio +from minio.credentials import AssumeRoleProvider, Credentials + +client = Minio('localhost:9000', + access_key='YOUR-ACCESSKEYID', + secret_key='YOUR-SECRETACCESSKEY' + ) + +restricted_upload_policy = """{ + "Version": "2012-10-17", + "Statement": [ + { + "Action": [ + "s3:PutObject" + ], + "Effect": "Allow", + "Resource": [ + "arn:aws:s3:::uploads/2020/*" + ], + "Sid": "Upload-access-to-specific-bucket-only" + } + ] +} +""" + +credentials_provider = AssumeRoleProvider(client, Policy=restricted_upload_policy) +temp_creds = Credentials(provider=credentials_provider) + +# User can access the credentials for e.g. serialization +print("Retrieved temporary credentials:") +print(temp_creds.get().access_key) +print(temp_creds.get().secret_key) + +# Initialize Minio client with the temporary credentials +restricted_client = Minio('localhost:9000', credentials=temp_creds) diff --git a/minio/compat.py b/minio/compat.py index 2811d1e54..701e7d5cb 100644 --- a/minio/compat.py +++ b/minio/compat.py @@ -40,7 +40,7 @@ from Queue import Empty queue_empty = Empty - from urllib import quote, unquote + from urllib import quote, unquote, urlencode from urlparse import urlsplit, parse_qs @@ -62,8 +62,9 @@ from queue import Empty queue_empty = Empty - from urllib.parse import quote, unquote, urlsplit, parse_qs + from urllib.parse import quote, unquote, urlsplit, parse_qs, urlencode unquote = unquote # to get rid of F401 + urlencode = urlencode # to get rid of F401 urlsplit = urlsplit # to get rid of F401 parse_qs = parse_qs # to get rid of F401 diff --git a/minio/credentials/__init__.py b/minio/credentials/__init__.py index 33691886b..36044f6ef 100644 --- a/minio/credentials/__init__.py +++ b/minio/credentials/__init__.py @@ -1,6 +1,7 @@ from .static import Static -from .credentials import Credentials +from .credentials import Credentials, Value from .chain import Chain +from .assume_role import AssumeRoleProvider from .aws_iam import IamEc2MetaData from .env_aws import EnvAWS from .env_minio import EnvMinio diff --git a/minio/credentials/assume_role.py b/minio/credentials/assume_role.py new file mode 100644 index 000000000..05dd10bfe --- /dev/null +++ b/minio/credentials/assume_role.py @@ -0,0 +1,94 @@ +# -*- coding: utf-8 -*- +# MinIO Python Library for Amazon S3 Compatible Cloud Storage, (C) +# 2020 MinIO, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from datetime import datetime + +from minio.compat import urlencode +from minio.error import ResponseError +from minio.credentials import Credentials +from minio.helpers import get_sha256_hexdigest +from minio.signer import sign_v4 + +from .credentials import Expiry, Provider +from .parsers import parse_assume_role + + +class AssumeRoleProvider(Provider): + region = 'us-east-1' + + # AWS STS support GET and POST requests for all actions. That is, the API does not require you to + # use GET for some actions and POST for others. However, GET requests are subject to the limitation + # size of a URL; although this limit is browser dependent, a typical limit is 2048 bytes. Therefore, + # for Query API requests that require larger sizes, you must use a POST request. + method = 'POST' + + def __init__(self, mc, RoleArn=None, RoleSessionName=None, Policy=None, DurationSeconds=None): + self._minio_client = mc + self._expiry = Expiry() + self._DurationSeconds = DurationSeconds + self._RoleArn = "arn:xxx:xxx:xxx:xxxx" if RoleArn is None else RoleArn + self._RoleSessionName = "anything" if RoleSessionName is None else RoleSessionName + self._Policy = Policy + + super(Provider, self).__init__() + + def retrieve(self): + + query = { + "Action": "AssumeRole", + "Version": "2011-06-15", + "RoleArn": self._RoleArn, + "RoleSessionName": self._RoleSessionName, + } + + # Add optional elements to the request + if self._Policy is not None: + query["Policy"] = self._Policy + + if self._DurationSeconds is not None: + query["DurationSeconds"] = str(self._DurationSeconds) + + url = self._minio_client._endpoint_url + "/" + content = urlencode(query) + headers = { + 'Content-Type': 'application/x-www-form-urlencoded; charset=utf-8', + 'User-Agent': self._minio_client._user_agent + } + + # Create signature headers + content_sha256_hex = get_sha256_hexdigest(content) + signed_headers = sign_v4(self.method, url, self.region, headers, + self._minio_client._credentials, + content_sha256=content_sha256_hex, + request_datetime=datetime.utcnow(), + service_name='sts' + ) + response = self._minio_client._http.urlopen(self.method, url, + body=content, + headers=signed_headers, + preload_content=True) + + if response.status != 200: + raise ResponseError(response, self.method).get_exception() + + # Parse the XML Response - getting the credentials as a Values instance. + credentials_value, expiry = parse_assume_role(response.data) + self._expiry.set_expiration(expiry) + + return credentials_value + + def is_expired(self): + return self._expiry.is_expired() + diff --git a/minio/credentials/aws_iam.py b/minio/credentials/aws_iam.py index abf5eddc5..8a9a4d4cd 100644 --- a/minio/credentials/aws_iam.py +++ b/minio/credentials/aws_iam.py @@ -17,8 +17,10 @@ import json import urllib3 import datetime + from .credentials import Provider, Value, Expiry from minio.error import ResponseError +from .parsers import parse_iam_credentials class IamEc2MetaData(Provider): @@ -56,7 +58,7 @@ def request_cred(self, creds_name): data = json.loads(res.data) if data['Code'] != 'Success': - raise ResponseError(res) + raise ResponseError(res, 'GET') return data @@ -67,15 +69,11 @@ def retrieve(self): creds_name = role_names[0] role_creds = self.request_cred(creds_name) - expiration = datetime.datetime.strptime( - role_creds['Expiration'], '%Y-%m-%dT%H:%M:%SZ') + credentials_value, expiration = parse_iam_credentials(role_creds) + self._expiry.set_expiration(expiration, self.default_expiry_window) - return Value( - access_key=role_creds['AccessKeyId'], - secret_key=role_creds['SecretAccessKey'], - session_token=role_creds['Token'] - ) + return credentials_value def is_expired(self): return self._expiry.is_expired() diff --git a/minio/credentials/credentials.py b/minio/credentials/credentials.py index f20bb9bd6..9bb9e9e82 100644 --- a/minio/credentials/credentials.py +++ b/minio/credentials/credentials.py @@ -17,6 +17,8 @@ from abc import ABCMeta, abstractmethod from datetime import datetime +import pytz + class Value(object): def __init__(self, access_key=None, secret_key=None, session_token=None): @@ -47,7 +49,8 @@ def set_expiration(self, expiration, time_delta=None): self._expiration = self._expiration + time_delta def is_expired(self): - return self._expiration < datetime.now() if self._expiration else True + utc_now = pytz.utc.localize(datetime.now()) + return self._expiration < utc_now if self._expiration else True class Credentials(object): diff --git a/minio/credentials/parsers.py b/minio/credentials/parsers.py new file mode 100644 index 000000000..79709495d --- /dev/null +++ b/minio/credentials/parsers.py @@ -0,0 +1,67 @@ +# -*- coding: utf-8 -*- +# MinIO Python Library for Amazon S3 Compatible Cloud Storage, (C) +# 2020 MinIO, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from xml.etree import ElementTree + +from .credentials import Value +from ..helpers import _iso8601_to_utc_datetime + +_XML_NS = { + 's3': 'http://s3.amazonaws.com/doc/2006-03-01/', + 'sts': 'https://sts.amazonaws.com/doc/2011-06-15/' +} + + +def parse_iam_credentials(data): + """ + Parser for IAM Instance Metadata Security Credentials. + + https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html + + :param data: Dict containing the json response. + :return: A 2-tuple containing: + - a :class:`~minio.credentials.Value` instance with the temporary credentials. + - A :class:`DateTime` instance of when the credentials expire. + """ + expiration = _iso8601_to_utc_datetime(data['Expiration']) + return Value( + access_key=data['AccessKeyId'], + secret_key=data['SecretAccessKey'], + session_token=data['Token'] + ), expiration + + +def parse_assume_role(data): + """ + Parser for assume role response. + + :param data: XML response data for STS assume role as a string. + :return: A 2-tuple containing: + - a :class:`~minio.credentials.Value` instance with the temporary credentials. + - A :class:`DateTime` instance of when the credentials expire. + """ + root = ElementTree.fromstring(data) + credentials_elem = root.find("sts:AssumeRoleResult", _XML_NS).find("sts:Credentials", _XML_NS) + + access_key = credentials_elem.find("sts:AccessKeyId", _XML_NS).text + secret_key = credentials_elem.find("sts:SecretAccessKey", _XML_NS).text + session_token = credentials_elem.find("sts:SessionToken", _XML_NS).text + + expiry_str = credentials_elem.find("sts:Expiration", _XML_NS).text + expiry = _iso8601_to_utc_datetime(expiry_str) + + return Value(access_key, secret_key, session_token), expiry + diff --git a/minio/error.py b/minio/error.py index 91f42967e..6a6ed4fec 100644 --- a/minio/error.py +++ b/minio/error.py @@ -179,15 +179,18 @@ def _set_error_response_with_body(self, bucket_name=None): raise InvalidXMLError('"Error" XML is not parsable. ' 'Message: {0}'.format(error)) + # Deal with namespaced response from sts + tag_prefix = '{https://sts.amazonaws.com/doc/2011-06-15/}' if root.tag == '{https://sts.amazonaws.com/doc/2011-06-15/}ErrorResponse' else '' + attrDict = { - 'Code': 'code', - 'BucketName': 'bucket_name', - 'Key': 'object_name', - 'Message': 'message', - 'RequestId': 'request_id', - 'HostId': 'host_id' + tag_prefix + 'Code': 'code', + tag_prefix + 'BucketName': 'bucket_name', + tag_prefix + 'Key': 'object_name', + tag_prefix + 'Message': 'message', + tag_prefix + 'RequestId': 'request_id', + tag_prefix + 'HostId': 'host_id' } - for attribute in root: + for attribute in root.iter(): attr = attrDict.get(attribute.tag) if attr: setattr(self, attr, attribute.text) diff --git a/minio/helpers.py b/minio/helpers.py index 173355994..d6680ff24 100644 --- a/minio/helpers.py +++ b/minio/helpers.py @@ -42,6 +42,9 @@ import os import errno import math +from datetime import datetime + +import pytz from .compat import (urlsplit, _quote, queryencode, str, bytes, basestring, _is_py3, _is_py2) @@ -755,3 +758,22 @@ def is_supported_header(key): # returns true if header is a storage class header def is_storageclass_header(key): return key.lower() == "x-amz-storage-class" + + +def _iso8601_to_utc_datetime(date_string): + """ + Convert iso8601 date string into UTC time. + + :param date_string: iso8601 formatted date string. + :return: :class:`datetime.datetime` with timezone set to UTC + """ + + # Handle timestamps with and without fractional seconds. Some non-AWS + # vendors (e.g. Dell EMC ECS) are not consistent about always providing + # fractional seconds. + try: + parsed_date = datetime.strptime(date_string, '%Y-%m-%dT%H:%M:%S.%fZ') + except ValueError: + parsed_date = datetime.strptime(date_string, '%Y-%m-%dT%H:%M:%SZ') + tz_aware_datetime = pytz.utc.localize(parsed_date) + return tz_aware_datetime diff --git a/minio/parsers.py b/minio/parsers.py index c30519bb4..2b5024751 100644 --- a/minio/parsers.py +++ b/minio/parsers.py @@ -30,7 +30,6 @@ from datetime import datetime # dependencies. -import pytz # minio specific. from .error import (ETREE_EXCEPTIONS, InvalidXMLError, MultiDeleteError) @@ -38,10 +37,14 @@ from .definitions import (Object, Bucket, IncompleteUpload, UploadPart, MultipartUploadResult, CopyObjectResult) +from .helpers import _iso8601_to_utc_datetime from .xml_marshal import (NOTIFICATIONS_ARN_FIELDNAME_MAP) -_S3_NS = {'s3': 'http://s3.amazonaws.com/doc/2006-03-01/'} +_XML_NS = { + 's3': 'http://s3.amazonaws.com/doc/2006-03-01/', + 'sts': 'https://sts.amazonaws.com/doc/2011-06-15/' +} class S3Element(object): @@ -78,14 +81,14 @@ def findall(self, name): """ return [ S3Element(self.root_name, elem) - for elem in self.element.findall('s3:{}'.format(name), _S3_NS) + for elem in self.element.findall('s3:{}'.format(name), _XML_NS) ] def find(self, name): """Similar to ElementTree.Element.find() """ - elt = self.element.find('s3:{}'.format(name), _S3_NS) + elt = self.element.find('s3:{}'.format(name), _XML_NS) return S3Element(self.root_name, elt) if elt else None def get_child_text(self, name, strict=True): @@ -96,14 +99,14 @@ def get_child_text(self, name, strict=True): """ if strict: try: - return self.element.find('s3:{}'.format(name), _S3_NS).text + return self.element.find('s3:{}'.format(name), _XML_NS).text except ETREE_EXCEPTIONS as error: raise InvalidXMLError( ('Invalid XML provided for "{}" - erroring tag <{}>. ' 'Message: {}').format(self.root_name, name, error) ) else: - return self.element.findtext('s3:{}'.format(name), None, _S3_NS) + return self.element.findtext('s3:{}'.format(name), None, _XML_NS) def get_urldecoded_elem_text(self, name, strict=True): """Like self.get_child_text(), but also performs urldecode() on the @@ -130,7 +133,7 @@ def get_localized_time_elem(self, name): """Parse a time XML child element. """ - return _iso8601_to_localized_time(self.get_child_text(name)) + return _iso8601_to_utc_datetime(self.get_child_text(name)) def text(self): """Fetch the current node's text @@ -355,25 +358,6 @@ def parse_location_constraint(data): return root.text() -def _iso8601_to_localized_time(date_string): - """ - Convert iso8601 date string into UTC time. - - :param date_string: iso8601 formatted date string. - :return: :class:`datetime.datetime` - """ - - # Handle timestamps with and without fractional seconds. Some non-AWS - # vendors (e.g. Dell EMC ECS) are not consistent about always providing - # fractional seconds. - try: - parsed_date = datetime.strptime(date_string, '%Y-%m-%dT%H:%M:%S.%fZ') - except ValueError: - parsed_date = datetime.strptime(date_string, '%Y-%m-%dT%H:%M:%SZ') - localized_time = pytz.utc.localize(parsed_date) - return localized_time - - def parse_get_bucket_notification(data): """ Parser for a get_bucket_notification response from S3. diff --git a/minio/signer.py b/minio/signer.py index a32e359b5..0863a67ad 100644 --- a/minio/signer.py +++ b/minio/signer.py @@ -336,7 +336,8 @@ def generate_credential_string(access_key, date, region, service_name=_DEFAULT_S def generate_authorization_header(access_key, date, region, - signed_headers, signature, service_name=_DEFAULT_SERVICE_NAME): + signed_headers, signature, + service_name=_DEFAULT_SERVICE_NAME): """ Generate authorization header.