Skip to content

Commit

Permalink
gh-37738: CI Build&Test: Show segfaults using GitHub annotations
Browse files Browse the repository at this point in the history
    
<!-- ^ Please provide a concise and informative title. -->
<!-- ^ Don't put issue numbers in the title, do this in the PR
description below. -->
<!-- ^ For example, instead of "Fixes #12345" use "Introduce new method
to calculate 1 + 2". -->
<!-- v Describe your changes below in detail. -->
<!-- v Why is this change required? What problem does it solve? -->
<!-- v If this PR resolves an open issue, please link to it here. For
example, "Fixes #12345". -->

Expanding the feature
https://github.com/sagemath/sage/wiki/Sage-10.3-Release-Tour#doctest-
failures-are-shown-in-the-files-changed-tab-on-prs
to cover doctests that crash the doctest runner.

An example (https://github.com/sagemath/sage/pull/37709/files)
<img width="1078" alt="Screenshot 2024-04-03 at 11 14 05 AM"
src="https://github.com/sagemath/sage/assets/8345221/de990f1f-069c-4fcf-
ac7e-ed915687533b">


### 📝 Checklist

<!-- Put an `x` in all the boxes that apply. -->

- [x] The title is concise and informative.
- [x] The description explains in detail what this PR is about.
- [ ] I have linked a relevant issue or discussion.
- [ ] I have created tests covering the changes.
- [ ] I have updated the documentation accordingly.

### ⌛ Dependencies

<!-- List all open PRs that this PR logically depends on. For example,
-->
<!-- - #12345: short description why this is a dependency -->
<!-- - #34567: ... -->
    
URL: #37738
Reported by: Matthias Köppe
Reviewer(s): Kwankyu Lee
  • Loading branch information
Release Manager committed May 1, 2024
2 parents 46b7ec2 + d605fa7 commit 9097fa2
Showing 1 changed file with 69 additions and 17 deletions.
86 changes: 69 additions & 17 deletions src/sage/doctest/reporting.py
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@
# https://www.gnu.org/licenses/
# ****************************************************************************

import re
from sys import stdout
from signal import (SIGABRT, SIGALRM, SIGBUS, SIGFPE, SIGHUP, SIGILL,
SIGINT, SIGKILL, SIGPIPE, SIGQUIT, SIGSEGV, SIGTERM)
Expand Down Expand Up @@ -218,6 +219,69 @@ def report_head(self, source, fail_msg=None):
cmd += f" [failed in baseline: {failed}]"
return cmd

def _log_failure(self, source, fail_msg, event, output=None):
r"""
Report on the result of a failed doctest run.
INPUT:
- ``source`` -- a source from :mod:`sage.doctest.sources`
- ``fail_msg`` -- a string
- ``event`` -- a string
- ``output`` -- optional string
EXAMPLES::
sage: from sage.doctest.reporting import DocTestReporter
sage: from sage.doctest.control import DocTestController, DocTestDefaults
sage: from sage.doctest.sources import FileDocTestSource
sage: from sage.env import SAGE_SRC
sage: import os
sage: filename = os.path.join(SAGE_SRC, 'sage', 'doctest', 'reporting.py')
sage: DD = DocTestDefaults()
sage: FDS = FileDocTestSource(filename, DD)
sage: DC = DocTestController(DD,[filename])
sage: DTR = DocTestReporter(DC)
sage: DTR._log_failure(FDS, "Timed out", "process (pid=1234) timed out", "Output so far...")
Timed out
**********************************************************************
Tests run before process (pid=1234) timed out:
Output so far...
**********************************************************************
"""
log = self.controller.log
format = self.controller.options.format
if format == 'sage':
stars = "*" * 70
log(f" {fail_msg}\n{stars}\n")
if output:
log(f"Tests run before {event}:")
log(output)
log(stars)
elif format == 'github':
# https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#using-workflow-commands-to-access-toolkit-functions
command = f'::error title={fail_msg}'
command += f',file={source.printpath}'
if output:
if m := re.search("## line ([0-9]+) ##\n-{40,100}\n(.*)", output, re.MULTILINE | re.DOTALL):
lineno = m.group(1)
message = m.group(2)
command += f',line={lineno}'
else:
message = output
# Urlencoding trick for multi-line annotations
# https://github.com/actions/starter-workflows/issues/68#issuecomment-581479448
message = message.replace('\n', '%0A')
else:
message = ""
command += f'::{message}'
log(command)
else:
raise ValueError(f'unknown format option: {format}')

def report(self, source, timeout, return_code, results, output, pid=None):
"""
Report on the result of running doctests on a given source.
Expand Down Expand Up @@ -435,9 +499,7 @@ def report(self, source, timeout, return_code, results, output, pid=None):
fail_msg += " (and interrupt failed)"
else:
fail_msg += " (with %s after interrupt)" % signal_name(sig)
log(" %s\n%s\nTests run before %s timed out:" % (fail_msg, "*"*70, process_name))
log(output)
log("*"*70)
self._log_failure(source, fail_msg, f"{process_name} timed out", output)
postscript['lines'].append(self.report_head(source, fail_msg))
stats[basename] = {"failed": True, "walltime": 1e6, "ntests": ntests}
if not baseline.get('failed', False):
Expand All @@ -449,9 +511,7 @@ def report(self, source, timeout, return_code, results, output, pid=None):
fail_msg = "Killed due to %s" % signal_name(-return_code)
if ntests > 0:
fail_msg += " after testing finished"
log(" %s\n%s\nTests run before %s failed:" % (fail_msg,"*"*70, process_name))
log(output)
log("*"*70)
self._log_failure(source, fail_msg, f"{process_name} failed", output)
postscript['lines'].append(self.report_head(source, fail_msg))
stats[basename] = {"failed": True, "walltime": 1e6, "ntests": ntests}
if not baseline.get('failed', False):
Expand All @@ -466,15 +526,11 @@ def report(self, source, timeout, return_code, results, output, pid=None):
else:
cpu = 1e6
if result_dict.err == 'badresult':
log(" Error in doctesting framework (bad result returned)\n%s\nTests run before error:" % ("*"*70))
log(output)
log("*"*70)
self._log_failure(source, "Error in doctesting framework (bad result returned)", "error", output)
postscript['lines'].append(self.report_head(source, "Testing error: bad result"))
self.error_status |= 64
elif result_dict.err == 'noresult':
log(" Error in doctesting framework (no result returned)\n%s\nTests run before error:" % ("*"*70))
log(output)
log("*"*70)
self._log_failure(source, "Error in doctesting framework (no result returned)", "error", output)
postscript['lines'].append(self.report_head(source, "Testing error: no result"))
self.error_status |= 64
elif result_dict.err == 'tab':
Expand All @@ -500,11 +556,7 @@ def report(self, source, timeout, return_code, results, output, pid=None):
else:
err = repr(result_dict.err)
fail_msg = "%s in doctesting framework" % err

log(" %s\n%s" % (fail_msg, "*"*70))
if output:
log("Tests run before doctest exception:\n" + output)
log("*"*70)
self._log_failure(source, fail_msg, "exception", output)
postscript['lines'].append(self.report_head(source, fail_msg))
if hasattr(result_dict, 'tb'):
log(result_dict.tb)
Expand Down

0 comments on commit 9097fa2

Please sign in to comment.