Add support for additional metadata headers.

docs/API.md

@@ -755,6 +755,7 @@ __Parameters__
 |``data``   |_io.RawIOBase_   |Any python object implementing io.RawIOBase. |
 |``length``   |_int_   |Total length of object.   |
 |``content_type``   |_string_ | Content type of the object. (optional, defaults to 'application/octet-stream').   |
+|``metadata``   |_dict_ | Any additional metadata. (optional, defaults to None). |
 
 __Return Value__
 
@@ -799,7 +800,8 @@ __Parameters__
 |``bucket_name``   |_string_  |Name of the bucket.   |
 |``object_name``   |_string_    |Name of the object.   |
 |``file_path``   |_string_ |Path on the local filesystem to which the object data will be written. |
-|``content_type``   |_string_ | Content type of the object. (optional, defaults to 'application/octet-stream').   |
+|``content_type``   |_string_ | Content type of the object. (optional, defaults to 'application/octet-stream'). |
+|``metadata``   |_dict_ | Any additional metadata. (optional, defaults to None). |
 
 __Return Value__
 

minio/fold_case_dict.py

@@ -0,0 +1,111 @@
+# -*- coding: utf-8 -*-
+# Minio Python Library for Amazon S3 Compatible Cloud Storage, (C)
+# 2017 Minio, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+minio.fold_case_dict
+
+This module implements a case insensitive dictionary.
+
+:copyright: (c) 2017 by Minio, Inc.
+:license: Apache 2.0, see LICENSE for more details.
+
+"""
+
+class FoldCaseDict():
+    def __init__(self, dictionary={}):
+        self._data = self.__create(dictionary)
+
+    def __create(self, value):
+        if isinstance(value, dict):
+            data = {}
+            for k, v in value.items():
+                if isinstance(v, dict):
+                    data[k.lower()] = FoldCaseDict(self.__create(v))
+                else:
+                    data[k.lower()] = v
+            return data
+        else:
+            return value
+
+    def __getitem__(self, item):
+        return self._data[item.lower()]
+
+    def __contains__(self, item):
+        return item.lower() in self._data
+
+    def __setitem__(self, key, value):
+        self._data[key.lower()] = self.__create(value)
+
+    def __delitem__(self, key):
+        del self._data[key.lower()]
+
+    def __iter__(self):
+        return (k for k in self._data.keys())
+
+    def __len__(self):
+        return len(self._data)
+
+    def __eq__(self, other):
+        if isinstance(other, dict):
+            other = FoldCaseDict(other)
+        elif isinstance(other, FoldCaseDict):
+            pass
+        else:
+            raise NotImplementedError
+
+        # Compare insensitively
+        return self.items() == other.items()
+
+    def __repr__(self):
+        return str(self._data)
+
+    def get(self, key, default=None):
+        if not key.lower() in self:
+            return default
+        else:
+            return self[key]
+
+    def has_key(self, key):
+        return key.lower() in self
+
+    def items(self):
+        return [(k, v) for k, v in self.iteritems()]
+
+    def keys(self):
+        return [k for k in self.iterkeys()]
+
+    def values(self):
+        return [v for v in self.itervalues()]
+
+    def iteritems(self):
+        for k, v in self._data.items():
+            yield k, v
+
+    def iterkeys(self):
+        for k, v in self._data.items():
+            yield k
+
+    def itervalues(self):
+        for k, v in self._data.items():
+            yield v
+
+    def update(self, dictionary):
+        if not (isinstance(dictionary, dict) or
+                isinstance(dictionary, FoldCaseDict)):
+            raise TypeError
+
+        for k, v in dictionary.items():
+            self[k] = v

minio/helpers.py

@@ -151,51 +151,6 @@ def parts_manager(data, part_size=5*1024*1024):
 
     return PartMetadata(tmpdata, md5hasher, sha256hasher, total_read)
 
-
-def ignore_headers(headers_to_sign):
-    """
-    Ignore headers.
-    """
-    # Excerpts from @lsegal -
-    # https://github.com/aws/aws-sdk-js/issues/659#issuecomment-120477258
-    #
-    #  User-Agent:
-    #
-    #      This is ignored from signing because signing this causes problems
-    #      with generating pre-signed URLs (that are executed by other agents)
-    #      or when customers pass requests through proxies, which may modify
-    #      the user-agent.
-    #
-    #  Content-Length:
-    #
-    #      This is ignored from signing because generating a pre-signed URL
-    #      should not provide a content-length constraint, specifically when
-    #      vending a S3 pre-signed PUT URL. The corollary to this is that when
-    #      sending regular requests (non-pre-signed), the signature contains
-    #      a checksum of the body, which implicitly validates the payload
-    #      length (since changing the number of bytes would change the
-    #      checksum) and therefore this header is not valuable in the
-    #      signature.
-    #
-    #  Content-Type:
-    #
-    #      Signing this header causes quite a number of problems in browser
-    #      environments, where browsers like to modify and normalize the
-    #      content-type header in different ways. There is more information
-    #      on this in https://github.com/aws/aws-sdk-js/issues/244. Avoiding
-    #      this field simplifies logic and reduces the possibility of bugs.
-    #
-    #  Authorization:
-    #
-    #      Is skipped for obvious reasons
-    ignored_headers = ['Authorization', 'Content-Length',
-                       'Content-Type', 'User-Agent']
-    for ignored_header in ignored_headers:
-        if ignored_header in headers_to_sign:
-            del headers_to_sign[ignored_header]
-
-    return headers_to_sign
-
 AWS_S3_ENDPOINT_MAP = {
     'us-east-1': 's3.amazonaws.com',
     'us-east-2': 's3-us-east-2.amazonaws.com',

minio/signer.py

@@ -32,7 +32,8 @@
 from datetime import datetime
 from .error import InvalidArgumentError
 from .compat import urlsplit, urlencode
-from .helpers import (ignore_headers, get_sha256_hexdigest)
+from .helpers import get_sha256_hexdigest
+from .fold_case_dict import FoldCaseDict
 
 # Signature version '4' algorithm.
 _SIGN_V4_ALGORITHM = 'AWS4-HMAC-SHA256'
@@ -92,14 +93,7 @@ def presign_v4(method, url, access_key, secret_key, region=None,
     date = datetime.utcnow()
     iso8601Date = date.strftime("%Y%m%dT%H%M%SZ")
 
-    headers_to_sign = dict(headers)
-
-    if response_headers is not None:
-        headers_to_sign.update(response_headers)
-
-    # Remove amazon recommended headers.
-    headers_to_sign = ignore_headers(headers)
-
+    headers_to_sign = headers
     # Construct queries.
     query = {}
     query['X-Amz-Algorithm'] = _SIGN_V4_ALGORITHM
@@ -140,6 +134,7 @@ def presign_v4(method, url, access_key, secret_key, region=None,
     canonical_request = generate_canonical_request(method,
                                                    new_parsed_url,
                                                    headers_to_sign,
+                                                   signed_headers,
                                                    content_hash_hex)
     string_to_sign = generate_string_to_sign(date, region,
                                              canonical_request)
@@ -184,7 +179,7 @@ def sign_v4(method, url, region, headers=None, access_key=None,
         return headers
 
     if headers is None:
-        headers = {}
+        headers = FoldCaseDict()
 
     if region is None:
         region = 'us-east-1'
@@ -204,15 +199,13 @@ def sign_v4(method, url, region, headers=None, access_key=None,
     headers['X-Amz-Date'] = date.strftime("%Y%m%dT%H%M%SZ")
     headers['X-Amz-Content-Sha256'] = content_sha256
 
-    headers_to_sign = dict(headers)
-
-    # Remove amazon recommended headers.
-    headers_to_sign = ignore_headers(headers_to_sign)
+    headers_to_sign = headers
 
     signed_headers = get_signed_headers(headers_to_sign)
     canonical_req = generate_canonical_request(method,
                                                parsed_url,
                                                headers_to_sign,
+                                               signed_headers,
                                                content_sha256)
     string_to_sign = generate_string_to_sign(date, region,
                                              canonical_req)
@@ -230,7 +223,7 @@ def sign_v4(method, url, region, headers=None, access_key=None,
     return headers
 
 
-def generate_canonical_request(method, parsed_url, headers, content_sha256):
+def generate_canonical_request(method, parsed_url, headers, signed_headers, content_sha256):
     """
     Generate canonical request.
 
@@ -251,13 +244,7 @@ def generate_canonical_request(method, parsed_url, headers, content_sha256):
     lines.append(query)
 
     # Headers added to canonical request.
-    signed_headers = []
     header_lines = []
-    for header in headers:
-        header = header.lower().strip()
-        signed_headers.append(header)
-    signed_headers = sorted(signed_headers)
-
     for header in signed_headers:
         value = headers[header.title()]
         value = str(value).strip()

tests/functional/tests.py

@@ -54,12 +54,13 @@ def main():
     bucket_name = 'minio-pytest'
 
     client.make_bucket(bucket_name)
-    if client._endpoint_url.startswith("s3.amazonaws"):
+    is_s3 = client._endpoint_url.startswith("s3.amazonaws")
+    if is_s3:
         client.make_bucket(bucket_name+'.unique',
                            location='us-west-1')
 
     ## Check if return codes a valid from server.
-    if client._endpoint_url.startswith("s3.amazonaws"):
+    if is_s3:
         try:
             client.make_bucket(bucket_name+'.unique',
                                location='us-west-1')
@@ -72,7 +73,7 @@ def main():
 
     # Check if bucket was created properly.
     client.bucket_exists(bucket_name)
-    if client._endpoint_url.startswith("s3.amazonaws"):
+    if is_s3:
         client.bucket_exists(bucket_name+'.unique')
 
     # List all buckets.
@@ -93,6 +94,9 @@ def main():
 
     # Fput a file
     client.fput_object(bucket_name, object_name+'-f', 'testfile')
+    if is_s3:
+        client.fput_object(bucket_name, object_name+'-f', 'testfile',
+                           metadata={'x-amz-storage-class': 'STANDARD_IA'})
 
     # Copy a file
     client.copy_object(bucket_name, object_name+'-copy',

tests/unit/minio_mocks.py

@@ -15,6 +15,7 @@
 # limitations under the License.
 
 from minio.compat import _is_py3
+from minio.fold_case_dict import FoldCaseDict
 
 if _is_py3:
     import http.client as httplib
@@ -28,7 +29,9 @@ def __init__(self, method, url, headers, status_code, response_headers=None,
                  content=None):
         self.method = method
         self.url = url
-        self.request_headers = headers
+        self.request_headers = FoldCaseDict()
+        for header in headers:
+            self.request_headers[header] = headers[header]
         self.status = status_code
         self.headers = response_headers
         self.data = content
@@ -42,7 +45,8 @@ def read(self, amt=1024):
     def mock_verify(self, method, url, headers):
         eq_(self.method, method)
         eq_(self.url, url)
-        eq_(self.request_headers, headers)
+        for header in headers:
+            eq_(self.request_headers[header], headers[header])
 
     # noinspection PyUnusedLocal
     def stream(self, chunk_size=1, decode_unicode=False):

tests/unit/sign_test.py

@@ -25,6 +25,7 @@
                           presign_v4)
 from minio.error import InvalidArgumentError
 from minio.compat import urlsplit
+from minio.fold_case_dict import FoldCaseDict
 
 empty_hash = 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855'
 dt = datetime(2015, 6, 20, 1, 2, 3, 0, pytz.utc)
@@ -38,12 +39,14 @@ def test_simple_request(self):
                                   empty_hash, 'x-amz-date:dateString',
                                   '', ';'.join(expected_signed_headers),
                                   empty_hash]
+        headers_to_sign = FoldCaseDict({'X-Amz-Date': 'dateString',
+                                        'X-Amz-Content-Sha256': empty_hash})
 
         expected_request = '\n'.join(expected_request_array)
         actual_request = generate_canonical_request('PUT',
                                                     url,
-                                                    {'X-Amz-Date': 'dateString',
-                                                     'X-Amz-Content-Sha256': empty_hash},
+                                                    headers_to_sign,
+                                                    expected_signed_headers,
                                                     empty_hash)
 
         eq_(expected_request, actual_request)
@@ -59,10 +62,13 @@ def test_request_with_query(self):
 
         expected_request = '\n'.join(expected_request_array)
 
+        headers_to_sign = FoldCaseDict({'X-Amz-Date': 'dateString',
+                                        'X-Amz-Content-Sha256': empty_hash})
+
         actual_request = generate_canonical_request('PUT',
                                                     url,
-                                                    {'X-Amz-Date': 'dateString',
-                                                     'X-Amz-Content-Sha256': empty_hash},
+                                                    headers_to_sign,
+                                                    expected_signed_headers,
                                                     empty_hash)
 
         eq_(expected_request, actual_request)