Skip to content
Permalink
Browse files
Fix flaky python-tests due to connection getting closed
Patch by David Capwell; reviewed by Caleb Rackliffe for CASSANDRA-17594
  • Loading branch information
dcapwell committed May 4, 2022
1 parent 3801a9b commit be48c417e6bbbd7c58f43ed001c9a2209a6339ab
Showing 4 changed files with 32 additions and 13 deletions.
@@ -219,15 +219,16 @@ def fixture_log_test_name_and_date(request, fixture_logging_setup):
def _filter_errors(dtest_setup, errors):
"""Filter errors, removing those that match ignore_log_patterns in the current DTestSetup"""
for e in errors:
e = repr(e)
for pattern in dtest_setup.ignore_log_patterns:
if re.search(pattern, repr(e)):
if re.search(pattern, e) or re.search(pattern, e.replace('\n', ' ')):
break
else:
yield e


def check_logs_for_errors(dtest_setup):
errors = []
all_errors = []
for node in dtest_setup.cluster.nodelist():
if not os.path.exists(node.logfilename()):
continue
@@ -240,11 +241,8 @@ def check_logs_for_errors(dtest_setup):
error_str = error.strip()

if error_str:
logger.error("Unexpected error in {node_name} log, error: \n{error}"
.format(node_name=node.name, error=error_str))
errors.append(error_str)
break
return errors
all_errors.append("[{node_name}] {error}".format(node_name=node.name, error=error_str))
return all_errors


def copy_logs(request, cluster, directory=None, name=None):
@@ -44,13 +44,18 @@ def retry_till_success(fun, *args, **kwargs):
# brief pause before next attempt
time.sleep(0.25)

def default_ignore_log_patterns():
# to allow tests to append to the list, make sure to create a new list as the output
# to this function, else multiple tests could corrupt the default set
return ['.*\[epollEventLoopGroup-.*\].*- Unknown exception in client networking.*: Connection reset by peer']


class DTestSetup(object):
def __init__(self, dtest_config=None, setup_overrides=None, cluster_name="test"):
self.dtest_config = dtest_config
self.setup_overrides = setup_overrides
self.cluster_name = cluster_name
self.ignore_log_patterns = []
self._ignore_log_patterns = default_ignore_log_patterns()
self.cluster = None
self.cluster_options = []
self.replacement_node = None
@@ -72,6 +77,23 @@ def __init__(self, dtest_config=None, setup_overrides=None, cluster_name="test")
self.create_cluster_func = None
self.iterations = 0

def set_ignore_log_patterns(self, other):
if self._ignore_log_patterns == None:
self._ignore_log_patterns = default_ignore_log_patterns()
# iteration is used here to allow [] and () patterns to work... anything iterable is allowed
for a in other:
self._ignore_log_patterns.append(a)

def get_ignore_log_patterns(self):
if self._ignore_log_patterns == None:
self._ignore_log_patterns = default_ignore_log_patterns()
return self._ignore_log_patterns

def del_ignore_log_patterns(self):
del self._ignore_log_patterns

ignore_log_patterns = property(get_ignore_log_patterns, set_ignore_log_patterns, del_ignore_log_patterns)

def get_test_path(self):
test_path = tempfile.mkdtemp(prefix='dtest-')

@@ -289,11 +311,9 @@ def check_logs_for_errors(self):

def __filter_errors(self, errors):
"""Filter errors, removing those that match self.ignore_log_patterns"""
if not hasattr(self, 'ignore_log_patterns'):
self.ignore_log_patterns = []
for e in errors:
for pattern in self.ignore_log_patterns:
if re.search(pattern, e):
if re.search(pattern, e) or re.search(pattern, e.replace('\n', ' ')):
break
else:
yield e
@@ -1267,7 +1267,8 @@ def _test_failure_during_repair(self, phase, initiator=False):
"Sync failed between .* and .*",
"failed to send a stream message/file to peer",
"failed to send a stream message/data to peer",
"Remote peer .* failed stream session"
"Remote peer .* failed stream session",
"org.apache.cassandra.repair.SomeRepairFailedException: null"
]

# stream session will be closed upon EOF, see CASSANDRA-15666
@@ -218,7 +218,7 @@ def test_resumable_decommission(self):
self.fixture_dtest_setup.ignore_log_patterns = [r'Streaming error occurred',
r'Error while decommissioning node',
r'Remote peer 127.0.0.2 failed stream session',
r'Remote peer 127.0.0.2:7000 failed stream session']
r'Remote peer \/?127.0.0.2:7000 failed stream session']
cluster = self.cluster
cluster.set_configuration_options(values={'stream_throughput_outbound_megabits_per_sec': 1})
cluster.populate(3, install_byteman=True).start()

0 comments on commit be48c41

Please sign in to comment.