Skip to content
Permalink
Browse files
CASSANDRA-16729 - user_types_test.py::TestUserTypes::test_type_keyspa…
…ce_permission_isolation uses retries to avoid flakyness

patch by David Capwell; reviewed by Brandon Williams for CASSANDRA-16729
  • Loading branch information
dcapwell authored and David Capwell committed Jun 10, 2021
1 parent 964051b commit f9ff078c3f54119ef8c7aa22987dd33f57d56de2
Showing 2 changed files with 37 additions and 9 deletions.
@@ -1,3 +1,8 @@
import logging
import time

logger = logging.getLogger(__name__)

class RerunTestException(Exception):
"""
This exception can be raised to signal a likely harmless test problem. If fixing a test is reasonable, that should be preferred.
@@ -30,3 +35,18 @@ def requires_rerun(err, *args):
"""
# err[0] contains the type of the error that occurred
return err[0] == RerunTestException

def retry(fn, num_retries=10, allowed_error=None, sleep_seconds=1):
last_error = None
for x in range(0, num_retries):
try:
return fn()
except Exception as e:
last_error = e
if allowed_error and not allowed_error(e):
break
logger.info("Retrying as error '{}' was seen; sleeping for {} seconds".format(str(e), str(sleep_seconds)))
time.sleep(sleep_seconds)
raise last_error


@@ -5,11 +5,12 @@

from flaky import flaky

from cassandra import ConsistencyLevel, Unauthorized
from cassandra import ConsistencyLevel, Unauthorized, InvalidRequest
from cassandra.query import SimpleStatement

from dtest import Tester, create_ks
from tools.assertions import assert_invalid
from tools.flaky import retry
from plugins.assert_tools import assert_regexp_matches

since = pytest.mark.since
@@ -529,17 +530,24 @@ def test_type_keyspace_permission_isolation(self):
cluster.set_configuration_options(values=config)
cluster.populate(3).start()
node1, node2, node3 = cluster.nodelist()
# need a bit of time for user to be created and propagate
time.sleep(5)

# do setup that requires a super user
superuser_session = self.patient_cql_connection(node1, user='cassandra', password='cassandra')
superuser_session.execute("create user ks1_user with password 'cassandra' nosuperuser;")
superuser_session.execute("create user ks2_user with password 'cassandra' nosuperuser;")
create_ks(superuser_session, 'ks1', 2)
create_ks(superuser_session, 'ks2', 2)
superuser_session.execute("grant all permissions on keyspace ks1 to ks1_user;")
superuser_session.execute("grant all permissions on keyspace ks2 to ks2_user;")

# role setup is async and does not log when complete; the only way to know its done (on a single instance) is to get a successful return.
def allowed_error(error):
return isinstance(error, InvalidRequest) and "Cannot process role related query as the role manager isn't yet setup" in str(error)
def r(fn):
retry(fn, allowed_error=allowed_error, sleep_seconds=10)

# role setup process is local to each instance, so can't rely on the first query passing to say this is complete; node1 may be complete but not node2 and node3
# for this reason, need to be aggressive about retries
r(lambda: superuser_session.execute("create user ks1_user with password 'cassandra' nosuperuser;"))
r(lambda: superuser_session.execute("create user ks2_user with password 'cassandra' nosuperuser;"))
r(lambda: create_ks(superuser_session, 'ks1', 2))
r(lambda: create_ks(superuser_session, 'ks2', 2))
r(lambda: superuser_session.execute("grant all permissions on keyspace ks1 to ks1_user;"))
r(lambda: superuser_session.execute("grant all permissions on keyspace ks2 to ks2_user;"))

user1_session = self.patient_cql_connection(node1, user='ks1_user', password='cassandra')
user2_session = self.patient_cql_connection(node1, user='ks2_user', password='cassandra')

0 comments on commit f9ff078

Please sign in to comment.