add support for AssumeRole STS provider (#874)

Closes #871

examples/assume_role.py

@@ -0,0 +1,71 @@
+# -*- coding: utf-8 -*-
+# MinIO Python Library for Amazon S3 Compatible Cloud Storage, (C) 2020 MinIO, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#
+# AssumeRoleProvider will call the Simple Token Service (STS) to retrieve
+# temporary credentials.
+#
+# - You can't call AssumeRole as a root user on either MinIO or AWS.
+#   For MinIO add a non-root user using the minio client `mc`:
+#
+#     mc admin user add myminio YOUR-ACCESSKEYID YOUR-SECRETACCESSKEY
+#   On AWS you will need an IAM user with the sts:AssumeRole action allowed,
+#   and a target role.
+# - The credentials will be valid for between 15 minutes and 12 hours.
+# - An access policy can be applied to the temporary credentials. The
+#   resulting permissions are the intersection of the role's existing policy
+#   and the optionally provided policy. You cannot grant more permissions than
+#   those allowed by the policy of the role that is being assumed.
+# - YOUR-ACCESSKEYID and YOUR-SECRETACCESSKEY are
+#   dummy values, please replace them with original values.
+# - To use minio with AWS, the `Minio` client that is passed to the
+#   AssumeRoleProvider must have the endpoint 'sts.amazonaws.com', and the
+#   RoleARN argument must be provided.
+
+from minio import Minio
+from minio.credentials import AssumeRoleProvider, Credentials
+
+client = Minio('localhost:9000',
+               access_key='YOUR-ACCESSKEYID',
+               secret_key='YOUR-SECRETACCESSKEY'
+               )
+
+restricted_upload_policy = """{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Action": [
+        "s3:PutObject"
+      ],
+      "Effect": "Allow",
+      "Resource": [
+        "arn:aws:s3:::uploads/2020/*"
+      ],
+      "Sid": "Upload-access-to-specific-bucket-only"
+    }
+  ]
+} 
+"""
+
+credentials_provider = AssumeRoleProvider(client, Policy=restricted_upload_policy)
+temp_creds = Credentials(provider=credentials_provider)
+
+# User can access the credentials for e.g. serialization
+print("Retrieved temporary credentials:")
+print(temp_creds.get().access_key)
+print(temp_creds.get().secret_key)
+
+# Initialize Minio client with the temporary credentials
+restricted_client = Minio('localhost:9000', credentials=temp_creds)

minio/compat.py

@@ -40,7 +40,7 @@
     from Queue import Empty
     queue_empty = Empty
 
-    from urllib import quote, unquote
+    from urllib import quote, unquote, urlencode
 
     from urlparse import urlsplit, parse_qs
 
@@ -62,8 +62,9 @@
     from queue import Empty
     queue_empty = Empty
 
-    from urllib.parse import quote, unquote, urlsplit, parse_qs
+    from urllib.parse import quote, unquote, urlsplit, parse_qs, urlencode
     unquote = unquote  # to get rid of F401
+    urlencode = urlencode  # to get rid of F401
     urlsplit = urlsplit  # to get rid of F401
     parse_qs = parse_qs  # to get rid of F401
 

minio/credentials/__init__.py

@@ -1,6 +1,7 @@
 from .static import Static
-from .credentials import Credentials
+from .credentials import Credentials, Value
 from .chain import Chain
+from .assume_role import AssumeRoleProvider
 from .aws_iam import IamEc2MetaData
 from .env_aws import EnvAWS
 from .env_minio import EnvMinio

minio/credentials/assume_role.py

@@ -0,0 +1,94 @@
+# -*- coding: utf-8 -*-
+# MinIO Python Library for Amazon S3 Compatible Cloud Storage, (C)
+# 2020 MinIO, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from datetime import datetime
+
+from minio.compat import urlencode
+from minio.error import ResponseError
+from minio.credentials import Credentials
+from minio.helpers import get_sha256_hexdigest
+from minio.signer import sign_v4
+
+from .credentials import Expiry, Provider
+from .parsers import parse_assume_role
+
+
+class AssumeRoleProvider(Provider):
+    region = 'us-east-1'
+
+    # AWS STS support GET and POST requests for all actions. That is, the API does not require you to
+    # use GET for some actions and POST for others. However, GET requests are subject to the limitation
+    # size of a URL; although this limit is browser dependent, a typical limit is 2048 bytes. Therefore,
+    # for Query API requests that require larger sizes, you must use a POST request.
+    method = 'POST'
+
+    def __init__(self, mc, RoleArn=None, RoleSessionName=None, Policy=None, DurationSeconds=None):
+        self._minio_client = mc
+        self._expiry = Expiry()
+        self._DurationSeconds = DurationSeconds
+        self._RoleArn = "arn:xxx:xxx:xxx:xxxx" if RoleArn is None else RoleArn
+        self._RoleSessionName = "anything" if RoleSessionName is None else RoleSessionName
+        self._Policy = Policy
+
+        super(Provider, self).__init__()
+
+    def retrieve(self):
+
+        query = {
+            "Action": "AssumeRole",
+            "Version": "2011-06-15",
+            "RoleArn": self._RoleArn,
+            "RoleSessionName": self._RoleSessionName,
+        }
+
+        # Add optional elements to the request
+        if self._Policy is not None:
+            query["Policy"] = self._Policy
+
+        if self._DurationSeconds is not None:
+            query["DurationSeconds"] = str(self._DurationSeconds)
+
+        url = self._minio_client._endpoint_url + "/"
+        content = urlencode(query)
+        headers = {
+            'Content-Type': 'application/x-www-form-urlencoded; charset=utf-8',
+            'User-Agent': self._minio_client._user_agent
+        }
+
+        # Create signature headers
+        content_sha256_hex = get_sha256_hexdigest(content)
+        signed_headers = sign_v4(self.method, url, self.region, headers,
+                                 self._minio_client._credentials,
+                                 content_sha256=content_sha256_hex,
+                                 request_datetime=datetime.utcnow(),
+                                 service_name='sts'
+                                 )
+        response = self._minio_client._http.urlopen(self.method, url,
+                                                    body=content,
+                                                    headers=signed_headers,
+                                                    preload_content=True)
+
+        if response.status != 200:
+            raise ResponseError(response, self.method).get_exception()
+
+        # Parse the XML Response - getting the credentials as a Values instance.
+        credentials_value, expiry = parse_assume_role(response.data)
+        self._expiry.set_expiration(expiry)
+
+        return credentials_value
+
+    def is_expired(self):
+        return self._expiry.is_expired()
+

minio/credentials/aws_iam.py

@@ -17,8 +17,10 @@
 import json
 import urllib3
 import datetime
+
 from .credentials import Provider, Value, Expiry
 from minio.error import ResponseError
+from .parsers import parse_iam_credentials
 
 
 class IamEc2MetaData(Provider):
@@ -56,7 +58,7 @@ def request_cred(self, creds_name):
 
         data = json.loads(res.data)
         if data['Code'] != 'Success':
-            raise ResponseError(res)
+            raise ResponseError(res, 'GET')
 
         return data
 
@@ -67,15 +69,11 @@ def retrieve(self):
 
         creds_name = role_names[0]
         role_creds = self.request_cred(creds_name)
-        expiration = datetime.datetime.strptime(
-            role_creds['Expiration'], '%Y-%m-%dT%H:%M:%SZ')
+        credentials_value, expiration = parse_iam_credentials(role_creds)
+
         self._expiry.set_expiration(expiration, self.default_expiry_window)
 
-        return Value(
-            access_key=role_creds['AccessKeyId'],
-            secret_key=role_creds['SecretAccessKey'],
-            session_token=role_creds['Token']
-        )
+        return credentials_value
 
     def is_expired(self):
         return self._expiry.is_expired()

minio/credentials/credentials.py

@@ -17,6 +17,8 @@
 from abc import ABCMeta, abstractmethod
 from datetime import datetime
 
+import pytz
+
 
 class Value(object):
     def __init__(self, access_key=None, secret_key=None, session_token=None):
@@ -47,7 +49,8 @@ def set_expiration(self, expiration, time_delta=None):
             self._expiration = self._expiration + time_delta
 
     def is_expired(self):
-        return self._expiration < datetime.now() if self._expiration else True
+        utc_now = pytz.utc.localize(datetime.now())
+        return self._expiration < utc_now if self._expiration else True
 
 
 class Credentials(object):

minio/credentials/parsers.py

@@ -0,0 +1,67 @@
+# -*- coding: utf-8 -*-
+# MinIO Python Library for Amazon S3 Compatible Cloud Storage, (C)
+# 2020 MinIO, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from xml.etree import ElementTree
+
+from .credentials import Value
+from ..helpers import _iso8601_to_utc_datetime
+
+_XML_NS = {
+    's3': 'http://s3.amazonaws.com/doc/2006-03-01/',
+    'sts': 'https://sts.amazonaws.com/doc/2011-06-15/'
+}
+
+
+def parse_iam_credentials(data):
+    """
+    Parser for IAM Instance Metadata Security Credentials.
+
+    https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html
+
+    :param data: Dict containing the json response.
+    :return: A 2-tuple containing:
+        - a :class:`~minio.credentials.Value` instance with the temporary credentials.
+        - A :class:`DateTime` instance of when the credentials expire.
+    """
+    expiration = _iso8601_to_utc_datetime(data['Expiration'])
+    return Value(
+        access_key=data['AccessKeyId'],
+        secret_key=data['SecretAccessKey'],
+        session_token=data['Token']
+    ), expiration
+
+
+def parse_assume_role(data):
+    """
+    Parser for assume role response.
+
+    :param data: XML response data for STS assume role as a string.
+    :return: A 2-tuple containing:
+        - a :class:`~minio.credentials.Value` instance with the temporary credentials.
+        - A :class:`DateTime` instance of when the credentials expire.
+    """
+    root = ElementTree.fromstring(data)
+    credentials_elem = root.find("sts:AssumeRoleResult", _XML_NS).find("sts:Credentials", _XML_NS)
+
+    access_key = credentials_elem.find("sts:AccessKeyId", _XML_NS).text
+    secret_key = credentials_elem.find("sts:SecretAccessKey", _XML_NS).text
+    session_token = credentials_elem.find("sts:SessionToken", _XML_NS).text
+
+    expiry_str = credentials_elem.find("sts:Expiration", _XML_NS).text
+    expiry = _iso8601_to_utc_datetime(expiry_str)
+
+    return Value(access_key, secret_key, session_token), expiry
+

minio/error.py

@@ -179,15 +179,18 @@ def _set_error_response_with_body(self, bucket_name=None):
             raise InvalidXMLError('"Error" XML is not parsable. '
                                   'Message: {0}'.format(error))
 
+        # Deal with namespaced response from sts
+        tag_prefix = '{https://sts.amazonaws.com/doc/2011-06-15/}' if root.tag == '{https://sts.amazonaws.com/doc/2011-06-15/}ErrorResponse' else ''
+
         attrDict = {
-            'Code': 'code',
-            'BucketName': 'bucket_name',
-            'Key': 'object_name',
-            'Message': 'message',
-            'RequestId': 'request_id',
-            'HostId': 'host_id'
+            tag_prefix + 'Code': 'code',
+            tag_prefix + 'BucketName': 'bucket_name',
+            tag_prefix + 'Key': 'object_name',
+            tag_prefix + 'Message': 'message',
+            tag_prefix + 'RequestId': 'request_id',
+            tag_prefix + 'HostId': 'host_id'
         }
-        for attribute in root:
+        for attribute in root.iter():
             attr = attrDict.get(attribute.tag)
             if attr:
                 setattr(self, attr, attribute.text)

minio/helpers.py

@@ -42,6 +42,9 @@
 import os
 import errno
 import math
+from datetime import datetime
+
+import pytz
 
 from .compat import (urlsplit, _quote, queryencode,
                      str, bytes, basestring, _is_py3, _is_py2)
@@ -755,3 +758,22 @@ def is_supported_header(key):
 # returns true if header is a storage class header
 def is_storageclass_header(key):
     return key.lower() == "x-amz-storage-class"
+
+
+def _iso8601_to_utc_datetime(date_string):
+    """
+    Convert iso8601 date string into UTC time.
+
+    :param date_string: iso8601 formatted date string.
+    :return: :class:`datetime.datetime` with timezone set to UTC
+    """
+
+    # Handle timestamps with and without fractional seconds. Some non-AWS
+    # vendors (e.g. Dell EMC ECS) are not consistent about always providing
+    # fractional seconds.
+    try:
+        parsed_date = datetime.strptime(date_string, '%Y-%m-%dT%H:%M:%S.%fZ')
+    except ValueError:
+        parsed_date = datetime.strptime(date_string, '%Y-%m-%dT%H:%M:%SZ')
+    tz_aware_datetime = pytz.utc.localize(parsed_date)
+    return tz_aware_datetime