Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[replay] Handle intermittent signature generation failure #343

Merged
merged 1 commit into from
Jan 4, 2023
Merged

[replay] Handle intermittent signature generation failure #343

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 25 additions & 7 deletions grizzly/replay/replay.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -98,9 +98,23 @@ def signature(self):
return self._signature

@staticmethod
def check_match(signature, report, expect_hang):
def check_match(signature, report, expect_hang, check_failed):
"""Check if report matches signature.

Args:
signature (CrashSignature): Signature to match.
report (Report): Report to check matches signature.
expect_hang (bool): A hang is expected.
check_failed (bool): Check if report signature creation failed.

Returns:
bool: True if report matches signature otherwise False.
"""
if signature is None:
# Treat 'None' signature as a bucket if it's not a hang
if check_failed and not report.is_hang:
# treat failed signature creation as a match
return report.crash_signature is None
# treat 'None' signature as a bucket if it's not a hang
return not report.is_hang
if expect_hang and not report.is_hang:
# avoid catching other crashes with forgiving hang signatures
Expand Down Expand Up @@ -304,6 +318,8 @@ def run(
reports = {}
try:
sig_hash = Report.calc_hash(self._signature) if self._signature else None
# an attempt has been made to set self._signature
sig_set = self._signature is not None
test_count = len(testcases)
relaunch = min(self._relaunch, repeat)
runner = Runner(
Expand Down Expand Up @@ -404,21 +420,23 @@ def run(
not runner.startup_failure
and not self._any_crash
and not run_result.timeout
and self._signature is None
and not sig_set
):
assert not expect_hang
assert self._signature is None
LOG.debug("no signature given, using short sig %r", short_sig)
self._signature = report.crash_signature
if self._signature is None:
LOG.debug("failed to generate signature to use")
else:
sig_set = True
if self._signature is not None:
assert not sig_hash, "sig_hash should only be set once"
sig_hash = Report.calc_hash(self._signature)

# bucket result
if not runner.startup_failure and (
self._any_crash
or self.check_match(self._signature, report, expect_hang)
or self.check_match(
self._signature, report, expect_hang, sig_set
)
):
if sig_hash is not None:
LOG.debug("using signature hash (%s) to bucket", sig_hash)
Expand Down
28 changes: 17 additions & 11 deletions grizzly/replay/test_replay.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -817,24 +817,30 @@ def _save_logs_variation(result_logs, _meta=False):
"stderr_log, ignored, total, include_stack",
[
# match stack only
(["STDERR log\n", "STDERR log\n"], 0, 2, True),
(["STDERR log\n", "STDERR log\n"], 0, 2, [True] * 2),
# match stack only
(["STDERR log\n", "STDERR log\nAssertion failure: test\n"], 0, 2, True),
(["STDERR log\n", "STDERR log\nAssertion failure: test\n"], 0, 2, [True] * 2),
# match stack and assertion message
(["STDERR log\nAssertion failure: test\n", "STDERR log\n"], 1, 1, True),
(["STDERR log\nAssertion failure: test\n", "STDERR log\n"], 1, 1, [True] * 2),
# match stack and assertion message
(["Assertion failure: #1\n", "Assertion failure: #2\n"], 1, 1, True),
(["Assertion failure: 1\n", "Assertion failure: 2\n"], 1, 1, [True] * 2),
# match, no match, match
(["Assertion failure: #1\n", "foo\n", "Assertion failure: #1\n"], 1, 2, True),
# fail to create signature (missing stack)
(["STDERR log\n", "STDERR log\n"], 0, 2, False),
(["Assertion failure: 1\n", "a\n", "Assertion failure: 1\n"], 1, 2, [True] * 3),
# fail to create signature x2 (missing stack)
(["STDERR log\n", "STDERR log\n"], 0, 2, [False] * 2),
# fail to create signature, create signature
(["STDERR log\n", "STDERR log\n"], 1, 1, [False, True]),
# create signature, fail to create signature
(["STDERR log\n", "STDERR log\n"], 1, 1, [True, False]),
],
)
def test_replay_25(mocker, server, stderr_log, ignored, total, include_stack):
"""test ReplayManager.run() - no signature - match first result"""
# NOTE: this is similar to test_replay_14 but it is more of an integration test
# NOTE: this is similar to "no signature - use first crash" test
# but this is more of an integration test
iters = len(stderr_log)
assert iters == ignored + total, "test is broken"
assert iters == len(include_stack), "test is broken"
server.serve_path.return_value = (Served.ALL, ["index.html"])
target = mocker.Mock(spec_set=Target, binary="fake_bin", launch_timeout=30)
target.check_result.return_value = Result.FOUND
Expand All @@ -844,11 +850,10 @@ def _save_logs_variation(result_logs, _meta=False):
"""create logs"""
nonlocal stderr_log
nonlocal include_stack
assert stderr_log, "test is broken"
log_path = Path(result_logs)
(log_path / "log_stderr.txt").write_text(stderr_log.pop(0))
(log_path / "log_stdout.txt").write_text("STDOUT log\n")
if include_stack:
if include_stack.pop(0):
with (log_path / "log_asan_blah.txt").open("w") as log_fp:
log_fp.write("==1==ERROR: AddressSanitizer: ")
log_fp.write("SEGV on unknown address 0x0 (pc 0x0 bp 0x0 sp 0x0 T0)\n")
Expand All @@ -857,10 +862,11 @@ def _save_logs_variation(result_logs, _meta=False):

target.save_logs.side_effect = _save_logs_variation

has_sig = include_stack[0]
with TestCase("index.html", "redirect.html", "test-adapter") as testcase:
with ReplayManager([], server, target, relaunch=10) as replay:
results = replay.run([testcase], 10, min_results=2, repeat=iters)
if include_stack:
if has_sig:
assert replay.signature is not None
else:
assert replay.signature is None
Expand Down