Correctly determine keystone v3 endpoint

The auth_uri argument in the keystone_authtoken section of the
configuration can, depending on the authentication plugin in use,
specify the URL with or without a version. When a version is given,
it may be v2.0 or v3. And for some plugins this setting may not even be
used. To help reduce the coupling between heat and keystonemiddleware's
configuration, this change adds a new "auth_uri" setting in the
[clients_keystone] section of the configuration that can be used to define
the unversioned keystone endpoint that heat should use. The keystone
discovery service is used to obtain the v3 URL from this endpoint. If this
new configuration item isn't set, then the legacy behavior that derives
the v3 endpoint from the middleware's setting is used.

UpgradeImpact: heat.conf [clients_keystone] auth_uri should be set
               to the unversioned keystone endpoint for wait conditions
               and wait handles to continue working.
Change-Id: I57d9749bea0b5797a9fc786e8fe991bbc63301ef
Partial-Bug: #1446918

heat/common/auth_url.py

@@ -14,6 +14,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from keystoneclient import discover as ks_discover
 from oslo_config import cfg
 from oslo_utils import importutils
 from webob import exc
@@ -33,10 +34,18 @@ def _get_auth_url(self):
         if 'auth_uri' in self.conf:
             return self.conf['auth_uri']
         else:
-            # Import auth_token to have keystone_authtoken settings setup.
-            auth_token_module = 'keystonemiddleware.auth_token'
-            importutils.import_module(auth_token_module)
-            return cfg.CONF.keystone_authtoken.auth_uri
+            # Look for the keystone auth_uri in the configuration. First we
+            # check the [clients_keystone] section, and if it is not set we
+            # look in [keystone_authtoken]
+            if cfg.CONF.clients_keystone.auth_uri:
+                discover = ks_discover.Discover(
+                    auth_url=cfg.CONF.clients_keystone.auth_uri)
+                return discover.url_for('3.0')
+            else:
+                # Import auth_token to have keystone_authtoken settings setup.
+                auth_token_module = 'keystonemiddleware.auth_token'
+                importutils.import_module(auth_token_module)
+                return cfg.CONF.keystone_authtoken.auth_uri
 
     def _validate_auth_url(self, auth_url):
         """Validate auth_url to ensure it can be used."""

heat/common/config.py

@@ -294,6 +294,12 @@
                help=_('Optional heat url in format like'
                       ' http://0.0.0.0:8004/v1/%(tenant_id)s.'))]
 
+keystone_client_opts = [
+    cfg.StrOpt('auth_uri',
+               default='',
+               help=_('Unversioned keystone url in format like'
+                      ' http://0.0.0.0:5000.'))]
+
 client_http_log_debug_opts = [
     cfg.BoolOpt('http_log_debug',
                 default=False,
@@ -351,6 +357,7 @@ def list_opts():
         yield client_specific_group, clients_opts
 
     yield 'clients_heat', heat_client_opts
+    yield 'clients_keystone', keystone_client_opts
     yield 'clients_nova', client_http_log_debug_opts
     yield 'clients_cinder', client_http_log_debug_opts
 

heat/common/context.py

@@ -15,6 +15,7 @@
 from keystoneclient.auth.identity import access as access_plugin
 from keystoneclient.auth.identity import v3
 from keystoneclient.auth import token_endpoint
+from keystoneclient import discover as ks_discover
 from oslo_config import cfg
 from oslo_context import context
 from oslo_log import log as logging
@@ -121,14 +122,29 @@ def from_dict(cls, values):
         return cls(**values)
 
     @property
-    def _keystone_v3_endpoint(self):
+    def keystone_v3_endpoint(self):
         if self.auth_url:
-            auth_uri = self.auth_url
+            auth_uri = self.auth_url.replace('v2.0', 'v3')
         else:
-            importutils.import_module('keystonemiddleware.auth_token')
-            auth_uri = cfg.CONF.keystone_authtoken.auth_uri
-
-        return auth_uri.replace('v2.0', 'v3')
+            # Look for the keystone auth_uri in the configuration. First we
+            # check the [clients_keystone] section, and if it is not set we
+            # look in [keystone_authtoken]
+            if cfg.CONF.clients_keystone.auth_uri:
+                discover = ks_discover.Discover(
+                    auth_url=cfg.CONF.clients_keystone.auth_uri)
+                auth_uri = discover.url_for('3.0')
+            else:
+                # Import auth_token to have keystone_authtoken settings setup.
+                importutils.import_module('keystonemiddleware.auth_token')
+                if cfg.CONF.keystone_authtoken.auth_uri:
+                    auth_uri = cfg.CONF.keystone_authtoken.auth_uri.replace(
+                        'v2.0', 'v3')
+                else:
+                    LOG.error('Keystone API endpoint not provided. Set '
+                              'auth_uri in section [clients_keystone] '
+                              'of the configuration file.')
+                    raise exception.AuthorizationFailure()
+        return auth_uri
 
     def _create_auth_plugin(self):
         if self.trust_id:
@@ -139,30 +155,30 @@ def _create_auth_plugin(self):
             return v3.Password(username=username,
                                password=password,
                                user_domain_id='default',
-                               auth_url=self._keystone_v3_endpoint,
+                               auth_url=self.keystone_v3_endpoint,
                                trust_id=self.trust_id)
 
         if self.auth_token_info:
             auth_ref = access.AccessInfo.factory(body=self.auth_token_info,
                                                  auth_token=self.auth_token)
             return access_plugin.AccessInfoPlugin(
-                auth_url=self._keystone_v3_endpoint,
+                auth_url=self.keystone_v3_endpoint,
                 auth_ref=auth_ref)
 
         if self.auth_token:
             # FIXME(jamielennox): This is broken but consistent. If you
             # only have a token but don't load a service catalog then
             # url_for wont work. Stub with the keystone endpoint so at
             # least it might be right.
-            return token_endpoint.Token(endpoint=self._keystone_v3_endpoint,
+            return token_endpoint.Token(endpoint=self.keystone_v3_endpoint,
                                         token=self.auth_token)
 
         if self.password:
             return v3.Password(username=self.username,
                                password=self.password,
                                project_id=self.tenant_id,
                                user_domain_id='default',
-                               auth_url=self._keystone_v3_endpoint)
+                               auth_url=self.keystone_v3_endpoint)
 
         LOG.error(_LE("Keystone v3 API connection failed, no password "
                       "trust or auth_token!"))

heat/common/heat_keystoneclient.py

@@ -76,14 +76,7 @@ def __init__(self, context):
         self._domain_admin_client = None
 
         self.session = session.Session.construct(self._ssl_options())
-
-        if self.context.auth_url:
-            self.v3_endpoint = self.context.auth_url.replace('v2.0', 'v3')
-        else:
-            # Import auth_token to have keystone_authtoken settings setup.
-            importutils.import_module('keystonemiddleware.auth_token')
-            self.v3_endpoint = cfg.CONF.keystone_authtoken.auth_uri.replace(
-                'v2.0', 'v3')
+        self.v3_endpoint = self.context.keystone_v3_endpoint
 
         if self.context.trust_id:
             # Create a client with the specified trust_id, this

heat/tests/test_auth_url.py

@@ -40,9 +40,27 @@ def setUp(self):
         self.config = {'auth_uri': 'foobar'}
         self.middleware = auth_url.AuthUrlFilter(self.app, self.config)
 
+    @mock.patch.object(auth_url.cfg, 'CONF')
+    def test_adds_default_auth_url_from_clients_keystone(self, mock_cfg):
+        self.config = {}
+        mock_cfg.clients_keystone.auth_uri = 'foobar'
+        mock_cfg.keystone_authtoken.auth_uri = 'this-should-be-ignored'
+        mock_cfg.auth_password.multi_cloud = False
+        with mock.patch('keystoneclient.discover.Discover') as discover:
+            class MockDiscover(object):
+                def url_for(self, endpoint):
+                    return 'foobar/v3'
+            discover.return_value = MockDiscover()
+            self.middleware = auth_url.AuthUrlFilter(self.app, self.config)
+            req = webob.Request.blank('/tenant_id/')
+            self.middleware(req)
+            self.assertIn('X-Auth-Url', req.headers)
+            self.assertEqual('foobar/v3', req.headers['X-Auth-Url'])
+
     @mock.patch.object(auth_url.cfg, 'CONF')
     def test_adds_default_auth_url_from_keystone_authtoken(self, mock_cfg):
         self.config = {}
+        mock_cfg.clients_keystone.auth_uri = ''
         mock_cfg.keystone_authtoken.auth_uri = 'foobar'
         mock_cfg.auth_password.multi_cloud = False
         self.middleware = auth_url.AuthUrlFilter(self.app, self.config)

heat/tests/test_common_context.py

@@ -17,6 +17,7 @@
 from oslo_config import cfg
 from oslo_middleware import request_id
 from oslo_policy import opts as policy_opts
+from oslo_utils import importutils
 import webob
 
 from heat.common import context
@@ -112,6 +113,66 @@ def test_admin_context_policy_false(self):
             ctx = context.RequestContext(roles=['notadmin'])
             self.assertFalse(ctx.is_admin)
 
+    def test_keystone_v3_endpoint_in_context(self):
+        """Ensure that the context is the preferred source for the
+        auth_uri.
+        """
+        cfg.CONF.set_override('auth_uri', 'http://xyz',
+                              group='clients_keystone')
+        policy_check = 'heat.common.policy.Enforcer.check_is_admin'
+        with mock.patch(policy_check) as pc:
+            pc.return_value = False
+            ctx = context.RequestContext(
+                auth_url='http://example.com:5000/v2.0')
+            self.assertEqual(ctx.keystone_v3_endpoint,
+                             'http://example.com:5000/v3')
+
+    def test_keystone_v3_endpoint_in_clients_keystone_config(self):
+        """Ensure that the [clients_keystone] section of the configuration is
+        the preferred source when the context does not have the auth_uri.
+        """
+        cfg.CONF.set_override('auth_uri', 'http://xyz',
+                              group='clients_keystone')
+        importutils.import_module('keystonemiddleware.auth_token')
+        cfg.CONF.set_override('auth_uri', 'http://abc/v2.0',
+                              group='keystone_authtoken')
+        policy_check = 'heat.common.policy.Enforcer.check_is_admin'
+        with mock.patch(policy_check) as pc:
+            pc.return_value = False
+            with mock.patch('keystoneclient.discover.Discover') as discover:
+                class MockDiscover(object):
+                    def url_for(self, endpoint):
+                        return 'http://xyz/v3'
+                discover.return_value = MockDiscover()
+
+                ctx = context.RequestContext(auth_url=None)
+                self.assertEqual(ctx.keystone_v3_endpoint, 'http://xyz/v3')
+
+    def test_keystone_v3_endpoint_in_keystone_authtoken_config(self):
+        """Ensure that the [keystone_authtoken] section of the configuration
+        is used when the auth_uri is not defined in the context or the
+        [clients_keystone] section.
+        """
+        importutils.import_module('keystonemiddleware.auth_token')
+        cfg.CONF.set_override('auth_uri', 'http://abc/v2.0',
+                              group='keystone_authtoken')
+        policy_check = 'heat.common.policy.Enforcer.check_is_admin'
+        with mock.patch(policy_check) as pc:
+            pc.return_value = False
+            ctx = context.RequestContext(auth_url=None)
+            self.assertEqual(ctx.keystone_v3_endpoint, 'http://abc/v3')
+
+    def test_keystone_v3_endpoint_not_set_in_config(self):
+        """Ensure an exception is raised when the auth_uri cannot be obtained
+        from any source.
+        """
+        policy_check = 'heat.common.policy.Enforcer.check_is_admin'
+        with mock.patch(policy_check) as pc:
+            pc.return_value = False
+            ctx = context.RequestContext(auth_url=None)
+            self.assertRaises(exception.AuthorizationFailure, getattr, ctx,
+                              'keystone_v3_endpoint')
+
 
 class RequestContextMiddlewareTest(common.HeatTestCase):