Remove the sql token driver and uuid token provider

Both of these drivers were staged for removal in Rocky. Now that
Rocky is open for development we can remove them. This commit removes
just the bare-bones aspects of each. Subsequent patches will do the
following:

  - Remove test class that were only meant for sql or uuid scenarios
  - Refactor the notification framework to not hint at token storage
  - Refactor the token provider API interfaces to be simpler and
    cleaner
  - Remove the needs_persistence property from the token provider API
    and document the ability to push that logic into individual
    providers that require it
  - Return 403 Forbidden for all requests to fetch a revocation list
  - Remove the signing directory configuration options

These changes will result in simpler interfaces which will be
important for people implementing their own token providers and
storage layers.

bp removed-as-of-rocky

Change-Id: I76d5c29f6b1572ee3ec7f2b1af63ff31572de2ce

doc/source/admin/token-provider.rst

@@ -20,16 +20,7 @@ You can register your own token provider by configuring the following property:
   entry point for the token provider in the ``keystone.token.provider``
   namespace.
 
-Each token format uses different technologies to achieve various performance,
-scaling, and architectural requirements. The Identity service includes
-``fernet``, ``pkiz``, ``pki``, and ``uuid`` token providers.
-
-Below is the detailed list of the token formats:
-
-UUID
- ``uuid`` tokens must be persisted (using the back end specified in the
- ``[token] driver`` option), but do not require any extra configuration
- or setup.
+Below is the detailed list of the token formats supported by keystone.:
 
 Fernet
  ``fernet`` tokens do not need to be persisted at all, but require that you run
@@ -38,6 +29,5 @@ Fernet
 
 .. warning::
 
-    UUID and Fernet tokens are both bearer tokens. They
-    must be protected from unnecessary disclosure to prevent unauthorized
-    access.
+    Fernet tokens are bearer tokens. They must be protected from unnecessary
+    disclosure to prevent unauthorized access.

doc/source/admin/token-support-matrix.ini

@@ -55,7 +55,6 @@
 # features. This list only covers drivers that are in tree. Out of tree
 # drivers should maintain their own equivalent document, and merge it with this
 # when their code merges into core.
-driver-impl-uuid=UUID tokens
 driver-impl-fernet=Fernet tokens
 
 [operation.create_unscoped_token]
@@ -65,7 +64,6 @@ notes=All token providers must be capable of issuing tokens without an explicit
   scope of authorization.
 cli=openstack --os-username=<username> --os-user-domain-name=<domain>
   --os-password=<password> token issue
-driver-impl-uuid=complete
 driver-impl-fernet=complete
 
 [operation.create_system_token]
@@ -74,7 +72,6 @@ status=mandatory
 notes=All token providers must be capable of issuing system-scoped tokens.
 cli=openstack --os-username=<username> --os-user-domain-name=<domain>
   --os-system token issue
-driver-impl-uuid=complete
 driver-impl-fernet=complete
 
 [operation.create_project_scoped_token]
@@ -84,7 +81,6 @@ notes=All token providers must be capable of issuing project-scoped tokens.
 cli=openstack --os-username=<username> --os-user-domain-name=<domain>
   --os-password=<password> --os-project-name=<project>
   --os-project-domain-name=<domain> token issue
-driver-impl-uuid=complete
 driver-impl-fernet=complete
 
 [operation.create_domain_scoped_token]
@@ -94,7 +90,6 @@ notes=Domain-scoped tokens are not required for all use cases, and for some use
   cases, projects can be used instead.
 cli=openstack --os-username=<username> --os-user-domain-name=<domain>
   --os-password=<password> --os-domain-name=<domain> token issue
-driver-impl-uuid=complete
 driver-impl-fernet=complete
 
 [operation.create_trust_scoped_token]
@@ -104,15 +99,13 @@ notes=Tokens scoped to a trust convey only the user impersonation and
   project-based authorization attributes included in the delegation.
 cli=openstack --os-username=<username> --os-user-domain-name=<domain>
   --os-password=<password> --os-trust-id=<trust> token issue
-driver-impl-uuid=complete
 driver-impl-fernet=complete
 
 [operation.create_token_using_oauth]
 title=Create a token given an OAuth access token
 status=optional
 notes=OAuth access tokens can be exchanged for keystone tokens.
 cli=
-driver-impl-uuid=complete
 driver-impl-fernet=complete
 
 [operation.create_token_with_bind]
@@ -121,7 +114,6 @@ status=optional
 notes=Tokens can express a binding to an additional authentication method, such
   as kerberos or x509.
 cli=
-driver-impl-uuid=complete
 driver-impl-fernet=missing
 
 [operation.revoke_token]
@@ -132,7 +124,6 @@ notes=Tokens may be individually revoked, such as when a user logs out of
   single token may be revoked as a result of this operation (such as when the
   revoked token was previously used to create additional tokens).
 cli=openstack token revoke
-driver-impl-uuid=complete
 driver-impl-fernet=complete
 
 [feature.online_validation]
@@ -141,7 +132,6 @@ status=mandatory
 notes=Keystone must be able to validate the tokens that it issues when
   presented with a token that it previously issued.
 cli=
-driver-impl-uuid=complete
 driver-impl-fernet=complete
 
 [feature.offline_validation]
@@ -151,7 +141,6 @@ notes=Services using Keystone for authentication may want to validate tokens
   themselves, rather than calling back to keystone, in order to improve
   performance and scalability.
 cli=
-driver-impl-uuid=missing
 driver-impl-fernet=missing
 
 [feature.non_persistent]
@@ -162,5 +151,4 @@ notes=If a token format does not require persistence (such as to a SQL
   keystone can issue at once, and there is no need to perform clean up
   operations such as `keystone-manage token_flush`.
 cli=
-driver-impl-uuid=missing
 driver-impl-fernet=complete

keystone/server/backends.py

@@ -26,7 +26,6 @@
 from keystone import resource
 from keystone import revoke
 from keystone import token
-from keystone.token import persistence
 from keystone import trust
 
 
@@ -49,8 +48,7 @@ def load_backends():
                 identity.Manager, identity.ShadowUsersManager,
                 limit.Manager, oauth1.Manager, policy.Manager,
                 resource.Manager, revoke.Manager, assignment.RoleManager,
-                trust.Manager, token.provider.Manager,
-                persistence.PersistenceManager]
+                trust.Manager, token.provider.Manager]
 
     drivers = {d._provides_api: d() for d in managers}
 

keystone/tests/unit/test_backend_sql.py

@@ -12,11 +12,8 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
-import datetime
-import functools
 import uuid
 
-import freezegun
 import mock
 from oslo_db import exception as db_exception
 from oslo_db import options
@@ -43,9 +40,7 @@
 from keystone.tests.unit.limit import test_backends as limit_tests
 from keystone.tests.unit.policy import test_backends as policy_tests
 from keystone.tests.unit.resource import test_backends as resource_tests
-from keystone.tests.unit.token import test_backends as token_tests
 from keystone.tests.unit.trust import test_backends as trust_tests
-from keystone.token.persistence.backends import sql as token_sql
 from keystone.trust.backends import sql as trust_sql
 
 
@@ -745,132 +740,6 @@ def test_trust_expires_at_int_matches_expires_at(self):
             self.assertEqual(trust_ref.expires_at, trust_ref.expires_at_int)
 
 
-class SqlToken(SqlTests, token_tests.TokenTests):
-    def test_token_revocation_list_uses_right_columns(self):
-        # This query used to be heavy with too many columns. We want
-        # to make sure it is only running with the minimum columns
-        # necessary.
-
-        expected_query_args = (token_sql.TokenModel.id,
-                               token_sql.TokenModel.expires,
-                               token_sql.TokenModel.extra,)
-
-        with mock.patch.object(token_sql, 'sql') as mock_sql:
-            tok = token_sql.Token()
-            tok.list_revoked_tokens()
-
-        mock_query = mock_sql.session_for_read().__enter__().query
-        mock_query.assert_called_with(*expected_query_args)
-
-    def test_flush_expired_tokens_batch(self):
-        # TODO(dstanek): This test should be rewritten to be less
-        # brittle. The code will likely need to be changed first. I
-        # just copied the spirit of the existing test when I rewrote
-        # mox -> mock. These tests are brittle because they have the
-        # call structure for SQLAlchemy encoded in them.
-
-        # test sqlite dialect
-        with mock.patch.object(token_sql, 'sql') as mock_sql:
-            mock_sql.get_session().bind.dialect.name = 'sqlite'
-            tok = token_sql.Token()
-            tok.flush_expired_tokens()
-
-        filter_mock = mock_sql.get_session().query().filter()
-        self.assertFalse(filter_mock.limit.called)
-        self.assertTrue(filter_mock.delete.called_once)
-
-    def test_flush_expired_tokens_batch_mysql(self):
-        # test mysql dialect, we don't need to test IBM DB SA separately, since
-        # other tests below test the differences between how they use the batch
-        # strategy
-        with mock.patch.object(token_sql, 'sql') as mock_sql:
-            mock_sql.session_for_write().__enter__(
-            ).query().filter().delete.return_value = 0
-
-            mock_sql.session_for_write().__enter__(
-            ).bind.dialect.name = 'mysql'
-
-            tok = token_sql.Token()
-            expiry_mock = mock.Mock()
-            ITERS = [1, 2, 3]
-            expiry_mock.return_value = iter(ITERS)
-            token_sql._expiry_range_batched = expiry_mock
-            tok.flush_expired_tokens()
-
-            # The expiry strategy is only invoked once, the other calls are via
-            # the yield return.
-            self.assertEqual(1, expiry_mock.call_count)
-
-            mock_delete = mock_sql.session_for_write().__enter__(
-            ).query().filter().delete
-
-            self.assertThat(mock_delete.call_args_list,
-                            matchers.HasLength(len(ITERS)))
-
-    def test_expiry_range_batched(self):
-        upper_bound_mock = mock.Mock(side_effect=[1, "final value"])
-        sess_mock = mock.Mock()
-        query_mock = sess_mock.query().filter().order_by().offset().limit()
-        query_mock.one.side_effect = [['test'], sql.NotFound()]
-        for i, x in enumerate(token_sql._expiry_range_batched(sess_mock,
-                                                              upper_bound_mock,
-                                                              batch_size=50)):
-            if i == 0:
-                # The first time the batch iterator returns, it should return
-                # the first result that comes back from the database.
-                self.assertEqual('test', x)
-            elif i == 1:
-                # The second time, the database range function should return
-                # nothing, so the batch iterator returns the result of the
-                # upper_bound function
-                self.assertEqual("final value", x)
-            else:
-                self.fail("range batch function returned more than twice")
-
-    def test_expiry_range_strategy_sqlite(self):
-        tok = token_sql.Token()
-        sqlite_strategy = tok._expiry_range_strategy('sqlite')
-        self.assertEqual(token_sql._expiry_range_all, sqlite_strategy)
-
-    def test_expiry_range_strategy_ibm_db_sa(self):
-        tok = token_sql.Token()
-        db2_strategy = tok._expiry_range_strategy('ibm_db_sa')
-        self.assertIsInstance(db2_strategy, functools.partial)
-        self.assertEqual(token_sql._expiry_range_batched, db2_strategy.func)
-        self.assertEqual({'batch_size': 100}, db2_strategy.keywords)
-
-    def test_expiry_range_strategy_mysql(self):
-        tok = token_sql.Token()
-        mysql_strategy = tok._expiry_range_strategy('mysql')
-        self.assertIsInstance(mysql_strategy, functools.partial)
-        self.assertEqual(token_sql._expiry_range_batched, mysql_strategy.func)
-        self.assertEqual({'batch_size': 1000}, mysql_strategy.keywords)
-
-    def test_expiry_range_with_allow_expired(self):
-        window_secs = 200
-        self.config_fixture.config(group='token',
-                                   allow_expired_window=window_secs)
-
-        tok = token_sql.Token()
-        time = datetime.datetime.utcnow()
-
-        with freezegun.freeze_time(time):
-            # unknown strategy just ensures we are getting the dumbest strategy
-            # that will remove everything in one go
-            strategy = tok._expiry_range_strategy('unkown')
-            upper_bound_func = token_sql._expiry_upper_bound_func
-
-            # session is ignored for dumb strategy
-            expiry_times = list(strategy(session=None,
-                                         upper_bound_func=upper_bound_func))
-
-            # basically just ensure that we are removing things in the past
-            delta = datetime.timedelta(seconds=window_secs)
-            previous_time = datetime.datetime.utcnow() - delta
-
-        self.assertEqual([previous_time], expiry_times)
-
-
 class SqlCatalog(SqlTests, catalog_tests.CatalogTests):
 
     _legacy_endpoint_id_in_endpoint = True
@@ -1047,24 +916,6 @@ class SqlImpliedRoles(SqlTests, assignment_tests.ImpliedRoleTests):
     pass
 
 
-class SqlTokenCacheInvalidationWithUUID(SqlTests,
-                                        token_tests.TokenCacheInvalidation):
-    def setUp(self):
-        super(SqlTokenCacheInvalidationWithUUID, self).setUp()
-        self._create_test_data()
-
-    def config_overrides(self):
-        super(SqlTokenCacheInvalidationWithUUID, self).config_overrides()
-        # NOTE(lbragstad): The TokenCacheInvalidation tests are coded to work
-        # against a persistent token backend. Only run these with token
-        # providers that issue persistent tokens.
-        self.config_fixture.config(group='token', provider='uuid')
-
-
-# NOTE(lbragstad): The Fernet token provider doesn't persist tokens in a
-# backend, so running the TokenCacheInvalidation tests here doesn't make sense.
-
-
 class SqlFilterTests(SqlTests, identity_tests.FilterTests):
 
     def clean_up_entities(self):

keystone/tests/unit/test_cli.py

@@ -53,23 +53,6 @@
 PROVIDERS = provider_api.ProviderAPIs
 
 
-class CliTestCase(unit.SQLDriverOverrides, unit.TestCase):
-    def config_files(self):
-        config_files = super(CliTestCase, self).config_files()
-        config_files.append(unit.dirs.tests_conf('backend_sql.conf'))
-        return config_files
-
-    def test_token_flush(self):
-        self.useFixture(database.Database())
-        self.load_backends()
-        # NOTE(morgan): we are testing a direct instantiation of the
-        # persistence manager for flushing. We should clear this out so we
-        # don't error. CLI should never impact a running service
-        # and should never actually lock the registry for dependencies.
-        provider_api.ProviderAPIs._clear_registry_instances()
-        cli.TokenFlush.main()
-
-
 class CliNoConfigTestCase(unit.BaseTestCase):
 
     def setUp(self):

keystone/tests/unit/test_revoke.py

@@ -483,15 +483,6 @@ def test_delete_group_without_role_does_not_revoke_users(self):
         self.assertEqual(2, len(revocation_backend.list_events()))
 
 
-class UUIDSqlRevokeTests(test_backend_sql.SqlTests, RevokeTests):
-    def config_overrides(self):
-        super(UUIDSqlRevokeTests, self).config_overrides()
-        self.config_fixture.config(
-            group='token',
-            provider='uuid',
-            revoke_by_id=False)
-
-
 class FernetSqlRevokeTests(test_backend_sql.SqlTests, RevokeTests):
     def config_overrides(self):
         super(FernetSqlRevokeTests, self).config_overrides()