Skip to content

Commit

Permalink
8287366: Improve test failure reporting in GHA
Browse files Browse the repository at this point in the history 
Backport-of: e0e15def24c4c93c863ff459788bea23ef99d790
  • Loading branch information
@RealCLanger
RealCLanger committed Jun 27, 2022
1 parent 8363d9d commit 1fbbd15
Showing 1 changed file with 285 additions and 21 deletions.
306 changes: 285 additions & 21 deletions .github/workflows/submit.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -364,15 +364,103 @@ jobs:
JTREG_KEYWORDS="!headful"
JTREG="JAVA_OPTIONS=-XX:-CreateCoredumpOnCrash"
- name: Check that all tests executed successfully
if: steps.run_tests.outcome != 'skipped'
run: >
if ! grep --include=test-summary.txt -lqr build/*/test-results -e "TEST SUCCESS" ; then
cat build/*/test-results/*/text/newfailures.txt ;
cat build/*/test-results/*/text/other_errors.txt ;
exit 1 ;
- name: Generate test failure summary
run: |
#
test_suite_name=$(cat build/run-test-prebuilt/test-support/test-last-ids.txt)
results_dir=build/run-test-prebuilt/test-results/$test_suite_name/text
failures=$(sed -e 's!\(.*\)\.java!\1!' -e '/^#/d' $results_dir/newfailures.txt || true)
errors=$(sed -e 's!\(.*\)\.java!\1!' -e '/^#/d' $results_dir/other_errors.txt || true)
failure_count=$(echo $failures | wc -w || true)
error_count=$(echo $errors | wc -w || true)
if [[ "$failures" = "" && "$errors" = "" ]]; then
# If we have nothing to report, exit this step now
exit 0
fi
echo "::error:: Test run reported $failure_count test failure(s) and $error_count error(s). See summary for details."
echo "### :boom: Test failures summary" >> $GITHUB_STEP_SUMMARY
if [[ "$failures" != "" ]]; then
echo "" >> $GITHUB_STEP_SUMMARY
echo "These tests reported failure:" >> $GITHUB_STEP_SUMMARY
for test in $failures; do
anchor="$(echo "$test" | tr [A-Z/] [a-z_])"
echo "* [$test](#user-content-$anchor)"
done >> $GITHUB_STEP_SUMMARY
fi
if [[ "$errors" != "" ]]; then
echo "" >> $GITHUB_STEP_SUMMARY
echo "These tests reported errors:" >> $GITHUB_STEP_SUMMARY
for test in $errors; do
anchor="$(echo "$test" | tr [A-Z/] [a-z_])"
echo "* [$test](#user-content-$anchor)"
done >> $GITHUB_STEP_SUMMARY
fi
- name: Collect failed test output
run: |
#
# This is a separate step, since if the markdown from a step gets bigger than
# 1024 kB it is skipped, but then the summary above is still generated
test_suite_name=$(cat build/run-test-prebuilt/test-support/test-last-ids.txt)
results_dir=build/run-test-prebuilt/test-results/$test_suite_name/text
report_dir=build/run-test-prebuilt/test-support/$test_suite_name
failures=$(sed -e 's!\(.*\)\.java!\1!' -e '/^#/d' $results_dir/newfailures.txt || true)
errors=$(sed -e 's!\(.*\)\.java!\1!' -e '/^#/d' $results_dir/other_errors.txt || true)
if [[ "$failures" = "" && "$errors" = "" ]]; then
# If we have nothing to report, exit this step now
exit 0
fi
echo "### Test output for failed tests" >> $GITHUB_STEP_SUMMARY
for test in $failures $errors; do
anchor="$(echo "$test" | tr [A-Z/] [a-z_])"
base_path="$(echo "$test" | tr '#' '_')"
report_file="$report_dir/$base_path.jtr"
hs_err_files="$report_dir/$base_path/hs_err*.log"
echo "#### <a id="$anchor">$test"
echo "<details><summary>View test results</summary>"
echo ""
echo '```'
if [[ -f "$report_file" ]]; then
cat "$report_file"
else
echo "Error: Result file $report_file not found"
fi
echo '```'
echo "</details>"
echo ""
if [[ "$hs_err_files" != "" ]]; then
echo "<details><summary>View HotSpot error log</summary>"
echo ""
for hs_err in $hs_err_files; do
echo '```'
echo "$hs_err:"
echo ""
cat "$hs_err"
echo '```'
done
echo "</details>"
echo ""
fi
done >> $GITHUB_STEP_SUMMARY
echo ":arrow_right: To see the entire test log, click the job in the list to the left" >> $GITHUB_STEP_SUMMARY
# This will abort the entire job in GHA, which is what we want
exit 1
- name: Create suitable test log artifact name
if: always()
run: echo "logsuffix=`echo ${{ matrix.test }} | sed -e 's!/!_!'g -e 's! !_!'g`" >> $GITHUB_ENV
Expand Down Expand Up @@ -834,15 +922,103 @@ jobs:
JTREG_KEYWORDS="!headful"
JTREG="JAVA_OPTIONS=-XX:-CreateCoredumpOnCrash"
- name: Check that all tests executed successfully
if: steps.run_tests.outcome != 'skipped'
run: >
if ! grep --include=test-summary.txt -lqr build/*/test-results -e "TEST SUCCESS" ; then
cat build/*/test-results/*/text/newfailures.txt ;
cat build/*/test-results/*/text/other_errors.txt ;
exit 1 ;
- name: Generate test failure summary
run: |
#
test_suite_name=$(cat build/run-test-prebuilt/test-support/test-last-ids.txt)
results_dir=build/run-test-prebuilt/test-results/$test_suite_name/text
failures=$(sed -e 's!\(.*\)\.java!\1!' -e '/^#/d' $results_dir/newfailures.txt || true)
errors=$(sed -e 's!\(.*\)\.java!\1!' -e '/^#/d' $results_dir/other_errors.txt || true)
failure_count=$(echo $failures | wc -w || true)
error_count=$(echo $errors | wc -w || true)
if [[ "$failures" = "" && "$errors" = "" ]]; then
# If we have nothing to report, exit this step now
exit 0
fi
echo "::error:: Test run reported $failure_count test failure(s) and $error_count error(s). See summary for details."
echo "### :boom: Test failures summary" >> $GITHUB_STEP_SUMMARY
if [[ "$failures" != "" ]]; then
echo "" >> $GITHUB_STEP_SUMMARY
echo "These tests reported failure:" >> $GITHUB_STEP_SUMMARY
for test in $failures; do
anchor="$(echo "$test" | tr [A-Z/] [a-z_])"
echo "* [$test](#user-content-$anchor)"
done >> $GITHUB_STEP_SUMMARY
fi
if [[ "$errors" != "" ]]; then
echo "" >> $GITHUB_STEP_SUMMARY
echo "These tests reported errors:" >> $GITHUB_STEP_SUMMARY
for test in $errors; do
anchor="$(echo "$test" | tr [A-Z/] [a-z_])"
echo "* [$test](#user-content-$anchor)"
done >> $GITHUB_STEP_SUMMARY
fi
- name: Collect failed test output
run: |
#
# This is a separate step, since if the markdown from a step gets bigger than
# 1024 kB it is skipped, but then the summary above is still generated
test_suite_name=$(cat build/run-test-prebuilt/test-support/test-last-ids.txt)
results_dir=build/run-test-prebuilt/test-results/$test_suite_name/text
report_dir=build/run-test-prebuilt/test-support/$test_suite_name
failures=$(sed -e 's!\(.*\)\.java!\1!' -e '/^#/d' $results_dir/newfailures.txt || true)
errors=$(sed -e 's!\(.*\)\.java!\1!' -e '/^#/d' $results_dir/other_errors.txt || true)
if [[ "$failures" = "" && "$errors" = "" ]]; then
# If we have nothing to report, exit this step now
exit 0
fi
echo "### Test output for failed tests" >> $GITHUB_STEP_SUMMARY
for test in $failures $errors; do
anchor="$(echo "$test" | tr [A-Z/] [a-z_])"
base_path="$(echo "$test" | tr '#' '_')"
report_file="$report_dir/$base_path.jtr"
hs_err_files="$report_dir/$base_path/hs_err*.log"
echo "#### <a id="$anchor">$test"
echo "<details><summary>View test results</summary>"
echo ""
echo '```'
if [[ -f "$report_file" ]]; then
cat "$report_file"
else
echo "Error: Result file $report_file not found"
fi
echo '```'
echo "</details>"
echo ""
if [[ "$hs_err_files" != "" ]]; then
echo "<details><summary>View HotSpot error log</summary>"
echo ""
for hs_err in $hs_err_files; do
echo '```'
echo "$hs_err:"
echo ""
cat "$hs_err"
echo '```'
done
echo "</details>"
echo ""
fi
done >> $GITHUB_STEP_SUMMARY
echo ":arrow_right: To see the entire test log, click the job in the list to the left" >> $GITHUB_STEP_SUMMARY
# This will abort the entire job in GHA, which is what we want
exit 1
- name: Create suitable test log artifact name
if: always()
run: echo "logsuffix=`echo ${{ matrix.test }} | sed -e 's!/!_!'g -e 's! !_!'g`" >> $GITHUB_ENV
Expand Down Expand Up @@ -1560,15 +1736,103 @@ jobs:
JTREG_KEYWORDS="!headful"
JTREG="JAVA_OPTIONS=-XX:-CreateCoredumpOnCrash"
- name: Check that all tests executed successfully
if: steps.run_tests.outcome != 'skipped'
run: >
if ! grep --include=test-summary.txt -lqr build/*/test-results -e "TEST SUCCESS" ; then
cat build/*/test-results/*/text/newfailures.txt ;
cat build/*/test-results/*/text/other_errors.txt ;
exit 1 ;
- name: Generate test failure summary
run: |
#
test_suite_name=$(cat build/run-test-prebuilt/test-support/test-last-ids.txt)
results_dir=build/run-test-prebuilt/test-results/$test_suite_name/text
failures=$(sed -e 's!\(.*\)\.java!\1!' -e '/^#/d' $results_dir/newfailures.txt || true)
errors=$(sed -e 's!\(.*\)\.java!\1!' -e '/^#/d' $results_dir/other_errors.txt || true)
failure_count=$(echo $failures | wc -w || true)
error_count=$(echo $errors | wc -w || true)
if [[ "$failures" = "" && "$errors" = "" ]]; then
# If we have nothing to report, exit this step now
exit 0
fi
echo "::error:: Test run reported $failure_count test failure(s) and $error_count error(s). See summary for details."
echo "### :boom: Test failures summary" >> $GITHUB_STEP_SUMMARY
if [[ "$failures" != "" ]]; then
echo "" >> $GITHUB_STEP_SUMMARY
echo "These tests reported failure:" >> $GITHUB_STEP_SUMMARY
for test in $failures; do
anchor="$(echo "$test" | tr [A-Z/] [a-z_])"
echo "* [$test](#user-content-$anchor)"
done >> $GITHUB_STEP_SUMMARY
fi
if [[ "$errors" != "" ]]; then
echo "" >> $GITHUB_STEP_SUMMARY
echo "These tests reported errors:" >> $GITHUB_STEP_SUMMARY
for test in $errors; do
anchor="$(echo "$test" | tr [A-Z/] [a-z_])"
echo "* [$test](#user-content-$anchor)"
done >> $GITHUB_STEP_SUMMARY
fi
- name: Collect failed test output
run: |
#
# This is a separate step, since if the markdown from a step gets bigger than
# 1024 kB it is skipped, but then the summary above is still generated
test_suite_name=$(cat build/run-test-prebuilt/test-support/test-last-ids.txt)
results_dir=build/run-test-prebuilt/test-results/$test_suite_name/text
report_dir=build/run-test-prebuilt/test-support/$test_suite_name
failures=$(sed -e 's!\(.*\)\.java!\1!' -e '/^#/d' $results_dir/newfailures.txt || true)
errors=$(sed -e 's!\(.*\)\.java!\1!' -e '/^#/d' $results_dir/other_errors.txt || true)
if [[ "$failures" = "" && "$errors" = "" ]]; then
# If we have nothing to report, exit this step now
exit 0
fi
echo "### Test output for failed tests" >> $GITHUB_STEP_SUMMARY
for test in $failures $errors; do
anchor="$(echo "$test" | tr [A-Z/] [a-z_])"
base_path="$(echo "$test" | tr '#' '_')"
report_file="$report_dir/$base_path.jtr"
hs_err_files="$report_dir/$base_path/hs_err*.log"
echo "#### <a id="$anchor">$test"
echo "<details><summary>View test results</summary>"
echo ""
echo '```'
if [[ -f "$report_file" ]]; then
cat "$report_file"
else
echo "Error: Result file $report_file not found"
fi
echo '```'
echo "</details>"
echo ""
if [[ "$hs_err_files" != "" ]]; then
echo "<details><summary>View HotSpot error log</summary>"
echo ""
for hs_err in $hs_err_files; do
echo '```'
echo "$hs_err:"
echo ""
cat "$hs_err"
echo '```'
done
echo "</details>"
echo ""
fi
done >> $GITHUB_STEP_SUMMARY
echo ":arrow_right: To see the entire test log, click the job in the list to the left" >> $GITHUB_STEP_SUMMARY
# This will abort the entire job in GHA, which is what we want
exit 1
- name: Create suitable test log artifact name
if: always()
run: echo "logsuffix=`echo ${{ matrix.test }} | sed -e 's!/!_!'g -e 's! !_!'g`" >> $GITHUB_ENV
Expand Down

1 comment on commit 1fbbd15

@openjdk-notifier
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Review

Issues

Please sign in to comment.