Stop using client headers for cross-middleware communication

Previously, we would use client-accessible headers to pass the S3 access
key, signature, and normalized request to authentication middleware.
Specifically, we would send the following headers:

    Authorization: AWS <access key>:<signature>
    X-Auth-Token: <base64-encoded normalized request>

However, few authentication middleware would validate that the
Authorization header actually started with "AWS ", the only prefix that
Swift3 would actually handle. As a result, the authentication
middlewares had no way to validate that the normalized request came from
swift3 rather than the client itself. This leads to a security hole
wherein an attacker who has captured a single valid request through the
S3 API or who has obtained a valid pre-signed URL may impersonate the
user that issued the request or pre-signed URL indefinitely through the
Swift API.

Now, the S3 authentication information will be placed in a separate
namespace in the WSGI environment, completely inaccessible to the
client. Specifically,

    environ['swift3.auth_details'] = {
        'access_key': <access key>,
        'signature': <signature>,
        'string_to_sign': <normalized request>,
    }

(Note that the normalized request is no longer base64-encoded.)

UpgradeImpact

This is a breaking API change. No currently-deployed authentication
middlewares will work with this. This patch includes a fix for s3_token
(used to authenticate against Keystone); any deployers still using
keystonemiddleware to provide s3_token should switch to using swift3.
Similar changes are being proposed for Swauth and tempauth. Proprietary
authentication middlewares will need to be updated to use the new
environment keys as well. When upgrading Swift3, operators will need to
upgrade their Swift3-capable authentication middleware at the same time.

Closes-Bug: 1561199
Change-Id: Ia3fbb4938f0daa8845cba4137a01cc43bc1a713c
Depends-On: Ib90adcc2f059adaf203fba1c95b2154561ea7487

swift3/request.py

@@ -13,7 +13,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import base64
 from email.header import Header
 from hashlib import sha256, md5
 import re
@@ -386,7 +385,12 @@ def __init__(self, env, app=None, slo_enabled=True):
         self.bucket_in_host = self._parse_host()
         self.container_name, self.object_name = self._parse_uri()
         self._validate_headers()
-        self.token = base64.urlsafe_b64encode(self._string_to_sign())
+        self.environ['swift3.auth_details'] = {
+            'access_key': self.access_key,
+            'signature': signature,
+            'string_to_sign': self._string_to_sign(),
+        }
+        self.token = None
         self.account = None
         self.user_id = None
         self.slo_enabled = slo_enabled
@@ -1199,13 +1203,15 @@ def authenticate(self, app):
                                       sw_resp.environ['HTTP_X_USER_NAME'])
             self.user_id = utf8encode(self.user_id)
             self.token = sw_resp.environ.get('HTTP_X_AUTH_TOKEN')
-            # Need to skip S3 authorization since authtoken middleware
-            # overwrites account in PATH_INFO
-            del self.headers['Authorization']
         else:
             # tempauth
             self.user_id = self.access_key
 
+        # Need to skip S3 authorization on subsequent requests to prevent
+        # overwriting the account in PATH_INFO
+        del self.headers['Authorization']
+        del self.environ['swift3.auth_details']
+
     def to_swift_req(self, method, container, obj, query=None,
                      body=None, headers=None):
         sw_req = super(S3AclRequest, self).to_swift_req(

swift3/s3_token_middleware.py

@@ -31,6 +31,7 @@
 
 """
 
+import base64
 import json
 import logging
 
@@ -176,31 +177,24 @@ def __call__(self, environ, start_response):
             return self._app(environ, start_response)
 
         # Read request signature and access id.
-        if 'Authorization' not in req.headers:
-            msg = 'No Authorization header. skipping.'
+        s3_auth_details = req.environ.get('swift3.auth_details')
+        if not s3_auth_details:
+            msg = 'No authorization deatils from Swift3. skipping.'
             self._logger.debug(msg)
             return self._app(environ, start_response)
 
-        token = req.headers.get('X-Auth-Token',
-                                req.headers.get('X-Storage-Token'))
-        if not token:
-            msg = 'You did not specify an auth or a storage token. skipping.'
-            self._logger.debug(msg)
-            return self._app(environ, start_response)
+        access = s3_auth_details['access_key']
+        if isinstance(access, six.binary_type):
+            access = access.decode('utf-8')
 
-        auth_header = req.headers['Authorization']
-        try:
-            access, signature = auth_header.split(' ')[-1].rsplit(':', 1)
-        except ValueError:
-            if self._delay_auth_decision:
-                self._logger.debug('Invalid Authorization header: %s - '
-                                   'deferring reject downstream', auth_header)
-                return self._app(environ, start_response)
-            else:
-                self._logger.debug('Invalid Authorization header: %s - '
-                                   'rejecting request', auth_header)
-                return self._deny_request('InvalidURI')(
-                    environ, start_response)
+        signature = s3_auth_details['signature']
+        if isinstance(signature, six.binary_type):
+            signature = signature.decode('utf-8')
+
+        string_to_sign = s3_auth_details['string_to_sign']
+        if isinstance(string_to_sign, six.text_type):
+            string_to_sign = string_to_sign.encode('utf-8')
+        token = base64.urlsafe_b64encode(string_to_sign).encode('ascii')
 
         # NOTE(chmou): This is to handle the special case with nova
         # when we have the option s3_affix_tenant. We will force it to

swift3/test/unit/test_middleware.py

@@ -18,7 +18,6 @@
 from contextlib import nested
 from datetime import datetime
 import hashlib
-import base64
 import requests
 import json
 import copy
@@ -102,16 +101,21 @@ def canonical_string(path, headers):
                 path, query_string = path.split('?', 1)
             else:
                 query_string = ''
+            env = {
+                'REQUEST_METHOD': 'GET',
+                'PATH_INFO': path,
+                'QUERY_STRING': query_string,
+                'HTTP_AUTHORIZATION': 'AWS X:Y:Z',
+            }
+            for header, value in headers.items():
+                header = 'HTTP_' + header.replace('-', '_').upper()
+                if header in ('HTTP_CONTENT_TYPE', 'HTTP_CONTENT_LENGTH'):
+                    header = header[5:]
+                env[header] = value
 
             with patch('swift3.request.Request._validate_headers'):
-                req = S3Request({
-                    'REQUEST_METHOD': 'GET',
-                    'PATH_INFO': path,
-                    'QUERY_STRING': query_string,
-                    'HTTP_AUTHORIZATION': 'AWS X:Y:Z',
-                })
-            req.headers.update(headers)
-            return req._string_to_sign()
+                req = S3Request(env)
+            return req.environ['swift3.auth_details']['string_to_sign']
 
         def verify(hash, path, headers):
             s = canonical_string(path, headers)
@@ -379,10 +383,12 @@ def test_token_generation(self):
         req.headers['Date'] = date_header
         status, headers, body = self.call_swift3(req)
         _, _, headers = self.swift.calls_with_headers[-1]
-        self.assertEqual(base64.urlsafe_b64decode(
-            headers['X-Auth-Token']),
-            'PUT\n\n\n%s\n/bucket/object?partNumber=1&uploadId=123456789abcdef'
-            % date_header)
+        self.assertEqual(req.environ['swift3.auth_details'], {
+            'access_key': 'test:tester',
+            'signature': 'hmac',
+            'string_to_sign': '\n'.join([
+                'PUT', '', '', date_header,
+                '/bucket/object?partNumber=1&uploadId=123456789abcdef'])})
 
     def test_invalid_uri(self):
         req = Request.blank('/bucket/invalid\xffname',

swift3/test/unit/test_obj.py

@@ -131,26 +131,51 @@ def test_object_HEAD_error(self):
         status, headers, body = self.call_swift3(req)
         self.assertEqual(status.split()[0], '403')
         self.assertEqual(body, '')  # sanity
+
+        req = Request.blank('/bucket/object',
+                            environ={'REQUEST_METHOD': 'HEAD'},
+                            headers={'Authorization': 'AWS test:tester:hmac',
+                                     'Date': self.get_date_header()})
         self.swift.register('HEAD', '/v1/AUTH_test/bucket/object',
                             swob.HTTPForbidden, {}, None)
         status, headers, body = self.call_swift3(req)
         self.assertEqual(status.split()[0], '403')
         self.assertEqual(body, '')  # sanity
+
+        req = Request.blank('/bucket/object',
+                            environ={'REQUEST_METHOD': 'HEAD'},
+                            headers={'Authorization': 'AWS test:tester:hmac',
+                                     'Date': self.get_date_header()})
         self.swift.register('HEAD', '/v1/AUTH_test/bucket/object',
                             swob.HTTPNotFound, {}, None)
         status, headers, body = self.call_swift3(req)
         self.assertEqual(status.split()[0], '404')
         self.assertEqual(body, '')  # sanity
+
+        req = Request.blank('/bucket/object',
+                            environ={'REQUEST_METHOD': 'HEAD'},
+                            headers={'Authorization': 'AWS test:tester:hmac',
+                                     'Date': self.get_date_header()})
         self.swift.register('HEAD', '/v1/AUTH_test/bucket/object',
                             swob.HTTPPreconditionFailed, {}, None)
         status, headers, body = self.call_swift3(req)
         self.assertEqual(status.split()[0], '412')
         self.assertEqual(body, '')  # sanity
+
+        req = Request.blank('/bucket/object',
+                            environ={'REQUEST_METHOD': 'HEAD'},
+                            headers={'Authorization': 'AWS test:tester:hmac',
+                                     'Date': self.get_date_header()})
         self.swift.register('HEAD', '/v1/AUTH_test/bucket/object',
                             swob.HTTPServerError, {}, None)
         status, headers, body = self.call_swift3(req)
         self.assertEqual(status.split()[0], '500')
         self.assertEqual(body, '')  # sanity
+
+        req = Request.blank('/bucket/object',
+                            environ={'REQUEST_METHOD': 'HEAD'},
+                            headers={'Authorization': 'AWS test:tester:hmac',
+                                     'Date': self.get_date_header()})
         self.swift.register('HEAD', '/v1/AUTH_test/bucket/object',
                             swob.HTTPServiceUnavailable, {}, None)
         status, headers, body = self.call_swift3(req)

swift3/test/unit/test_request.py

@@ -221,7 +221,7 @@ def create_s3request_with_param(param, value):
         self.assertEqual(
             result.exception.headers['content-type'], 'application/xml')
 
-    def test_authenticate_delete_Authorization_from_s3req_headers(self):
+    def test_authenticate_delete_Authorization_from_s3req(self):
         req = Request.blank('/bucket/obj',
                             environ={'REQUEST_METHOD': 'GET'},
                             headers={'Authorization': 'AWS test:tester:hmac',
@@ -232,11 +232,12 @@ def test_authenticate_delete_Authorization_from_s3req_headers(self):
 
             m_swift_resp.return_value = FakeSwiftResponse()
             s3_req = S3AclRequest(req.environ, MagicMock())
-            self.assertTrue('HTTP_AUTHORIZATION' not in s3_req.environ)
-            self.assertTrue('Authorization' not in s3_req.headers)
+            self.assertNotIn('swift3.auth_details', s3_req.environ)
+            self.assertNotIn('HTTP_AUTHORIZATION', s3_req.environ)
+            self.assertNotIn('Authorization', s3_req.headers)
             self.assertEqual(s3_req.token, 'token')
 
-    def test_to_swift_req_Authorization_not_exist_in_swreq_headers(self):
+    def test_to_swift_req_Authorization_not_exist_in_swreq(self):
         container = 'bucket'
         obj = 'obj'
         method = 'GET'
@@ -251,6 +252,7 @@ def test_to_swift_req_Authorization_not_exist_in_swreq_headers(self):
             m_swift_resp.return_value = FakeSwiftResponse()
             s3_req = S3AclRequest(req.environ, MagicMock())
             sw_req = s3_req.to_swift_req(method, container, obj)
+            self.assertNotIn('swift3.auth_details', sw_req.environ)
             self.assertNotIn('HTTP_AUTHORIZATION', sw_req.environ)
             self.assertNotIn('Authorization', sw_req.headers)
             self.assertEqual(sw_req.headers['X-Auth-Token'], 'token')