Skip to content

Commit

Permalink
8287366: Improve test failure reporting in GHA
Browse files Browse the repository at this point in the history
Reviewed-by: clanger
  • Loading branch information
magicus committed Jun 1, 2022
1 parent 7846971 commit e0e15de
Showing 1 changed file with 285 additions and 21 deletions.
306 changes: 285 additions & 21 deletions .github/workflows/submit.yml
Expand Up @@ -361,15 +361,103 @@ jobs:
JTREG_KEYWORDS="!headful"
JTREG="JAVA_OPTIONS=-XX:-CreateCoredumpOnCrash"
- name: Check that all tests executed successfully
if: steps.run_tests.outcome != 'skipped'
run: >
if ! grep --include=test-summary.txt -lqr build/*/test-results -e "TEST SUCCESS" ; then
cat build/*/test-results/*/text/newfailures.txt ;
cat build/*/test-results/*/text/other_errors.txt ;
exit 1 ;
- name: Generate test failure summary
run: |
#
test_suite_name=$(cat build/run-test-prebuilt/test-support/test-last-ids.txt)
results_dir=build/run-test-prebuilt/test-results/$test_suite_name/text
failures=$(sed -e 's!\(.*\)\.java!\1!' -e '/^#/d' $results_dir/newfailures.txt || true)
errors=$(sed -e 's!\(.*\)\.java!\1!' -e '/^#/d' $results_dir/other_errors.txt || true)
failure_count=$(echo $failures | wc -w || true)
error_count=$(echo $errors | wc -w || true)
if [[ "$failures" = "" && "$errors" = "" ]]; then
# If we have nothing to report, exit this step now
exit 0
fi
echo "::error:: Test run reported $failure_count test failure(s) and $error_count error(s). See summary for details."
echo "### :boom: Test failures summary" >> $GITHUB_STEP_SUMMARY
if [[ "$failures" != "" ]]; then
echo "" >> $GITHUB_STEP_SUMMARY
echo "These tests reported failure:" >> $GITHUB_STEP_SUMMARY
for test in $failures; do
anchor="$(echo "$test" | tr [A-Z/] [a-z_])"
echo "* [$test](#user-content-$anchor)"
done >> $GITHUB_STEP_SUMMARY
fi
if [[ "$errors" != "" ]]; then
echo "" >> $GITHUB_STEP_SUMMARY
echo "These tests reported errors:" >> $GITHUB_STEP_SUMMARY
for test in $errors; do
anchor="$(echo "$test" | tr [A-Z/] [a-z_])"
echo "* [$test](#user-content-$anchor)"
done >> $GITHUB_STEP_SUMMARY
fi
- name: Collect failed test output
run: |
#
# This is a separate step, since if the markdown from a step gets bigger than
# 1024 kB it is skipped, but then the summary above is still generated
test_suite_name=$(cat build/run-test-prebuilt/test-support/test-last-ids.txt)
results_dir=build/run-test-prebuilt/test-results/$test_suite_name/text
report_dir=build/run-test-prebuilt/test-support/$test_suite_name
failures=$(sed -e 's!\(.*\)\.java!\1!' -e '/^#/d' $results_dir/newfailures.txt || true)
errors=$(sed -e 's!\(.*\)\.java!\1!' -e '/^#/d' $results_dir/other_errors.txt || true)
if [[ "$failures" = "" && "$errors" = "" ]]; then
# If we have nothing to report, exit this step now
exit 0
fi
echo "### Test output for failed tests" >> $GITHUB_STEP_SUMMARY
for test in $failures $errors; do
anchor="$(echo "$test" | tr [A-Z/] [a-z_])"
base_path="$(echo "$test" | tr '#' '_')"
report_file="$report_dir/$base_path.jtr"
hs_err_files="$report_dir/$base_path/hs_err*.log"
echo "#### <a id="$anchor">$test"
echo "<details><summary>View test results</summary>"
echo ""
echo '```'
if [[ -f "$report_file" ]]; then
cat "$report_file"
else
echo "Error: Result file $report_file not found"
fi
echo '```'
echo "</details>"
echo ""
if [[ "$hs_err_files" != "" ]]; then
echo "<details><summary>View HotSpot error log</summary>"
echo ""
for hs_err in $hs_err_files; do
echo '```'
echo "$hs_err:"
echo ""
cat "$hs_err"
echo '```'
done
echo "</details>"
echo ""
fi
done >> $GITHUB_STEP_SUMMARY
echo ":arrow_right: To see the entire test log, click the job in the list to the left" >> $GITHUB_STEP_SUMMARY
# This will abort the entire job in GHA, which is what we want
exit 1
- name: Create suitable test log artifact name
if: always()
run: echo "logsuffix=`echo ${{ matrix.test }} | sed -e 's!/!_!'g -e 's! !_!'g`" >> $GITHUB_ENV
Expand Down Expand Up @@ -827,15 +915,103 @@ jobs:
JTREG_KEYWORDS="!headful"
JTREG="JAVA_OPTIONS=-XX:-CreateCoredumpOnCrash"
- name: Check that all tests executed successfully
if: steps.run_tests.outcome != 'skipped'
run: >
if ! grep --include=test-summary.txt -lqr build/*/test-results -e "TEST SUCCESS" ; then
cat build/*/test-results/*/text/newfailures.txt ;
cat build/*/test-results/*/text/other_errors.txt ;
exit 1 ;
- name: Generate test failure summary
run: |
#
test_suite_name=$(cat build/run-test-prebuilt/test-support/test-last-ids.txt)
results_dir=build/run-test-prebuilt/test-results/$test_suite_name/text
failures=$(sed -e 's!\(.*\)\.java!\1!' -e '/^#/d' $results_dir/newfailures.txt || true)
errors=$(sed -e 's!\(.*\)\.java!\1!' -e '/^#/d' $results_dir/other_errors.txt || true)
failure_count=$(echo $failures | wc -w || true)
error_count=$(echo $errors | wc -w || true)
if [[ "$failures" = "" && "$errors" = "" ]]; then
# If we have nothing to report, exit this step now
exit 0
fi
echo "::error:: Test run reported $failure_count test failure(s) and $error_count error(s). See summary for details."
echo "### :boom: Test failures summary" >> $GITHUB_STEP_SUMMARY
if [[ "$failures" != "" ]]; then
echo "" >> $GITHUB_STEP_SUMMARY
echo "These tests reported failure:" >> $GITHUB_STEP_SUMMARY
for test in $failures; do
anchor="$(echo "$test" | tr [A-Z/] [a-z_])"
echo "* [$test](#user-content-$anchor)"
done >> $GITHUB_STEP_SUMMARY
fi
if [[ "$errors" != "" ]]; then
echo "" >> $GITHUB_STEP_SUMMARY
echo "These tests reported errors:" >> $GITHUB_STEP_SUMMARY
for test in $errors; do
anchor="$(echo "$test" | tr [A-Z/] [a-z_])"
echo "* [$test](#user-content-$anchor)"
done >> $GITHUB_STEP_SUMMARY
fi
- name: Collect failed test output
run: |
#
# This is a separate step, since if the markdown from a step gets bigger than
# 1024 kB it is skipped, but then the summary above is still generated
test_suite_name=$(cat build/run-test-prebuilt/test-support/test-last-ids.txt)
results_dir=build/run-test-prebuilt/test-results/$test_suite_name/text
report_dir=build/run-test-prebuilt/test-support/$test_suite_name
failures=$(sed -e 's!\(.*\)\.java!\1!' -e '/^#/d' $results_dir/newfailures.txt || true)
errors=$(sed -e 's!\(.*\)\.java!\1!' -e '/^#/d' $results_dir/other_errors.txt || true)
if [[ "$failures" = "" && "$errors" = "" ]]; then
# If we have nothing to report, exit this step now
exit 0
fi
echo "### Test output for failed tests" >> $GITHUB_STEP_SUMMARY
for test in $failures $errors; do
anchor="$(echo "$test" | tr [A-Z/] [a-z_])"
base_path="$(echo "$test" | tr '#' '_')"
report_file="$report_dir/$base_path.jtr"
hs_err_files="$report_dir/$base_path/hs_err*.log"
echo "#### <a id="$anchor">$test"
echo "<details><summary>View test results</summary>"
echo ""
echo '```'
if [[ -f "$report_file" ]]; then
cat "$report_file"
else
echo "Error: Result file $report_file not found"
fi
echo '```'
echo "</details>"
echo ""
if [[ "$hs_err_files" != "" ]]; then
echo "<details><summary>View HotSpot error log</summary>"
echo ""
for hs_err in $hs_err_files; do
echo '```'
echo "$hs_err:"
echo ""
cat "$hs_err"
echo '```'
done
echo "</details>"
echo ""
fi
done >> $GITHUB_STEP_SUMMARY
echo ":arrow_right: To see the entire test log, click the job in the list to the left" >> $GITHUB_STEP_SUMMARY
# This will abort the entire job in GHA, which is what we want
exit 1
- name: Create suitable test log artifact name
if: always()
run: echo "logsuffix=`echo ${{ matrix.test }} | sed -e 's!/!_!'g -e 's! !_!'g`" >> $GITHUB_ENV
Expand Down Expand Up @@ -1664,15 +1840,103 @@ jobs:
JTREG_KEYWORDS="!headful"
JTREG="JAVA_OPTIONS=-XX:-CreateCoredumpOnCrash"
- name: Check that all tests executed successfully
if: steps.run_tests.outcome != 'skipped'
run: >
if ! grep --include=test-summary.txt -lqr build/*/test-results -e "TEST SUCCESS" ; then
cat build/*/test-results/*/text/newfailures.txt ;
cat build/*/test-results/*/text/other_errors.txt ;
exit 1 ;
- name: Generate test failure summary
run: |
#
test_suite_name=$(cat build/run-test-prebuilt/test-support/test-last-ids.txt)
results_dir=build/run-test-prebuilt/test-results/$test_suite_name/text
failures=$(sed -e 's!\(.*\)\.java!\1!' -e '/^#/d' $results_dir/newfailures.txt || true)
errors=$(sed -e 's!\(.*\)\.java!\1!' -e '/^#/d' $results_dir/other_errors.txt || true)
failure_count=$(echo $failures | wc -w || true)
error_count=$(echo $errors | wc -w || true)
if [[ "$failures" = "" && "$errors" = "" ]]; then
# If we have nothing to report, exit this step now
exit 0
fi
echo "::error:: Test run reported $failure_count test failure(s) and $error_count error(s). See summary for details."
echo "### :boom: Test failures summary" >> $GITHUB_STEP_SUMMARY
if [[ "$failures" != "" ]]; then
echo "" >> $GITHUB_STEP_SUMMARY
echo "These tests reported failure:" >> $GITHUB_STEP_SUMMARY
for test in $failures; do
anchor="$(echo "$test" | tr [A-Z/] [a-z_])"
echo "* [$test](#user-content-$anchor)"
done >> $GITHUB_STEP_SUMMARY
fi
if [[ "$errors" != "" ]]; then
echo "" >> $GITHUB_STEP_SUMMARY
echo "These tests reported errors:" >> $GITHUB_STEP_SUMMARY
for test in $errors; do
anchor="$(echo "$test" | tr [A-Z/] [a-z_])"
echo "* [$test](#user-content-$anchor)"
done >> $GITHUB_STEP_SUMMARY
fi
- name: Collect failed test output
run: |
#
# This is a separate step, since if the markdown from a step gets bigger than
# 1024 kB it is skipped, but then the summary above is still generated
test_suite_name=$(cat build/run-test-prebuilt/test-support/test-last-ids.txt)
results_dir=build/run-test-prebuilt/test-results/$test_suite_name/text
report_dir=build/run-test-prebuilt/test-support/$test_suite_name
failures=$(sed -e 's!\(.*\)\.java!\1!' -e '/^#/d' $results_dir/newfailures.txt || true)
errors=$(sed -e 's!\(.*\)\.java!\1!' -e '/^#/d' $results_dir/other_errors.txt || true)
if [[ "$failures" = "" && "$errors" = "" ]]; then
# If we have nothing to report, exit this step now
exit 0
fi
echo "### Test output for failed tests" >> $GITHUB_STEP_SUMMARY
for test in $failures $errors; do
anchor="$(echo "$test" | tr [A-Z/] [a-z_])"
base_path="$(echo "$test" | tr '#' '_')"
report_file="$report_dir/$base_path.jtr"
hs_err_files="$report_dir/$base_path/hs_err*.log"
echo "#### <a id="$anchor">$test"
echo "<details><summary>View test results</summary>"
echo ""
echo '```'
if [[ -f "$report_file" ]]; then
cat "$report_file"
else
echo "Error: Result file $report_file not found"
fi
echo '```'
echo "</details>"
echo ""
if [[ "$hs_err_files" != "" ]]; then
echo "<details><summary>View HotSpot error log</summary>"
echo ""
for hs_err in $hs_err_files; do
echo '```'
echo "$hs_err:"
echo ""
cat "$hs_err"
echo '```'
done
echo "</details>"
echo ""
fi
done >> $GITHUB_STEP_SUMMARY
echo ":arrow_right: To see the entire test log, click the job in the list to the left" >> $GITHUB_STEP_SUMMARY
# This will abort the entire job in GHA, which is what we want
exit 1
- name: Create suitable test log artifact name
if: always()
run: echo "logsuffix=`echo ${{ matrix.test }} | sed -e 's!/!_!'g -e 's! !_!'g`" >> $GITHUB_ENV
Expand Down

5 comments on commit e0e15de

@openjdk-notifier
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@RealCLanger
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

/backport jdk17u-dev

@RealCLanger
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

/backport jdk11u-dev

@openjdk
Copy link

@openjdk openjdk bot commented on e0e15de Jun 25, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@RealCLanger the backport was successfully created on the branch RealCLanger-backport-e0e15def in my personal fork of openjdk/jdk17u-dev. To create a pull request with this backport targeting openjdk/jdk17u-dev:master, just click the following link:

➡️ Create pull request

The title of the pull request is automatically filled in correctly and below you find a suggestion for the pull request body:

Hi all,

This pull request contains a backport of commit e0e15def from the openjdk/jdk repository.

The commit being backported was authored by Magnus Ihse Bursie on 1 Jun 2022 and was reviewed by Christoph Langer.

Thanks!

If you need to update the source branch of the pull then run the following commands in a local clone of your personal fork of openjdk/jdk17u-dev:

$ git fetch https://github.com/openjdk-bots/jdk17u-dev RealCLanger-backport-e0e15def:RealCLanger-backport-e0e15def
$ git checkout RealCLanger-backport-e0e15def
# make changes
$ git add paths/to/changed/files
$ git commit --message 'Describe additional changes made'
$ git push https://github.com/openjdk-bots/jdk17u-dev RealCLanger-backport-e0e15def

@openjdk
Copy link

@openjdk openjdk bot commented on e0e15de Jun 25, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@RealCLanger the backport was successfully created on the branch RealCLanger-backport-e0e15def in my personal fork of openjdk/jdk11u-dev. To create a pull request with this backport targeting openjdk/jdk11u-dev:master, just click the following link:

➡️ Create pull request

The title of the pull request is automatically filled in correctly and below you find a suggestion for the pull request body:

Hi all,

This pull request contains a backport of commit e0e15def from the openjdk/jdk repository.

The commit being backported was authored by Magnus Ihse Bursie on 1 Jun 2022 and was reviewed by Christoph Langer.

Thanks!

If you need to update the source branch of the pull then run the following commands in a local clone of your personal fork of openjdk/jdk11u-dev:

$ git fetch https://github.com/openjdk-bots/jdk11u-dev RealCLanger-backport-e0e15def:RealCLanger-backport-e0e15def
$ git checkout RealCLanger-backport-e0e15def
# make changes
$ git add paths/to/changed/files
$ git commit --message 'Describe additional changes made'
$ git push https://github.com/openjdk-bots/jdk11u-dev RealCLanger-backport-e0e15def

Please sign in to comment.