[ews] mac-wk2 build marked as green even though it found failure in s…

…tress mode

https://bugs.webkit.org/show_bug.cgi?id=260020

Reviewed by Jonathan Bedard.

Check if stress mode already set build_summary before updating it.
Also mark build as failure if stress mode detected failures.

* Tools/CISupport/ews-build/steps.py:
(RunWebKitTests.evaluateCommand):
(RunWebKitTestsInStressMode): Updated the failure text to be more clear.
(ReRunWebKitTests.evaluateCommand):
(AnalyzeLayoutTestsResults.report_pre_existing_failures):
(SetBuildSummary.start):
* Tools/CISupport/ews-build/steps_unittest.py:

Canonical link: https://commits.webkit.org/266824@main

Tools/CISupport/ews-build/steps.py

@@ -3481,17 +3481,20 @@ def evaluateResult(self, cmd):
 
     def evaluateCommand(self, cmd):
         rc = self.evaluateResult(cmd)
+        previous_build_summary = self.getProperty('build_summary', '')
         if rc == SUCCESS or rc == WARNINGS:
             message = 'Passed layout tests'
             self.descriptionDone = message
             self.build.results = SUCCESS
-            self.setProperty('build_summary', message)
+            if RunWebKitTestsInStressMode.FAILURE_MSG_IN_STRESS_MODE not in previous_build_summary:
+                self.setProperty('build_summary', message)
         elif (self.preexisting_failures_in_results_db and len(self.failing_tests_filtered) == 0):
             # This means all the tests which failed in this run were also failing or flaky in results database
             message = f"Ignored pre-existing failure: {', '.join(self.preexisting_failures_in_results_db)}"
             self.descriptionDone = message
             self.build.results = SUCCESS
-            self.setProperty('build_summary', message)
+            if RunWebKitTestsInStressMode.FAILURE_MSG_IN_STRESS_MODE not in previous_build_summary:
+                self.setProperty('build_summary', message)
             self.build.addStepsAfterCurrentStep([ArchiveTestResults(),
                                                 UploadTestResults(),
                                                 ExtractTestResults()])
@@ -3530,6 +3533,7 @@ class RunWebKitTestsInStressMode(RunWebKitTests):
     suffix = 'stress-mode'
     EXIT_AFTER_FAILURES = '10'
     ENABLE_ADDITIONAL_ARGUMENTS = False
+    FAILURE_MSG_IN_STRESS_MODE = 'Found test failures in stress mode'
 
     def __init__(self, num_iterations=100):
         self.num_iterations = num_iterations
@@ -3551,7 +3555,7 @@ def evaluateCommand(self, cmd):
             self.build.results = SUCCESS
             self.setProperty('build_summary', message)
         else:
-            self.setProperty('build_summary', 'Found test failures')
+            self.setProperty('build_summary', self.FAILURE_MSG_IN_STRESS_MODE)
             self.build.addStepsAfterCurrentStep([
                 ArchiveTestResults(),
                 UploadTestResults(identifier=self.suffix),
@@ -3585,6 +3589,7 @@ def evaluateCommand(self, cmd):
         num_flaky_failures = len(flaky_failures)
         flaky_failures = sorted(list(flaky_failures))[:self.NUM_FAILURES_TO_DISPLAY]
         flaky_failures_string = ', '.join(flaky_failures)
+        previous_build_summary = self.getProperty('build_summary', '')
 
         if rc == SUCCESS or rc == WARNINGS:
             message = 'Passed layout tests'
@@ -3595,13 +3600,15 @@ def evaluateCommand(self, cmd):
                 message = 'Found flaky test{}: {}'.format(pluralSuffix, flaky_failures_string)
                 for flaky_failure in flaky_failures:
                     self.send_email_for_flaky_failure(flaky_failure)
-            self.setProperty('build_summary', message)
+            if RunWebKitTestsInStressMode.FAILURE_MSG_IN_STRESS_MODE not in previous_build_summary:
+                self.setProperty('build_summary', message)
         elif (self.preexisting_failures_in_results_db and len(self.failing_tests_filtered) == 0):
             # This means all the tests which failed in this run were also failing or flaky in results database
             message = f"Ignored pre-existing failure: {', '.join(self.preexisting_failures_in_results_db)}"
             self.descriptionDone = message
             self.build.results = SUCCESS
-            self.setProperty('build_summary', message)
+            if RunWebKitTestsInStressMode.FAILURE_MSG_IN_STRESS_MODE not in previous_build_summary:
+                self.setProperty('build_summary', message)
             self.build.addStepsAfterCurrentStep([ArchiveTestResults(),
                                                 UploadTestResults(identifier='rerun'),
                                                 ExtractTestResults(identifier='rerun')])
@@ -3617,7 +3624,8 @@ def evaluateCommand(self, cmd):
                     self.send_email_for_flaky_failure(flaky_failure)
                 self.descriptionDone = message
                 self.build.results = SUCCESS
-                self.setProperty('build_summary', message)
+                if RunWebKitTestsInStressMode.FAILURE_MSG_IN_STRESS_MODE not in previous_build_summary:
+                    self.setProperty('build_summary', message)
                 self.build.addStepsAfterCurrentStep([ArchiveTestResults(),
                                                     UploadTestResults(identifier='rerun'),
                                                     ExtractTestResults(identifier='rerun')])
@@ -3776,6 +3784,7 @@ def report_failure(self, new_failures=None, exceed_failure_limit=False, failure_
     def report_pre_existing_failures(self, clean_tree_failures, flaky_failures):
         self.build.results = SUCCESS
         self.descriptionDone = 'Passed layout tests'
+        previous_build_summary = self.getProperty('build_summary', '')
         message = ''
         if clean_tree_failures:
             clean_tree_failures_string = ', '.join(sorted(clean_tree_failures)[:self.NUM_FAILURES_TO_DISPLAY])
@@ -3795,7 +3804,8 @@ def report_pre_existing_failures(self, clean_tree_failures, flaky_failures):
             for flaky_failure in list(flaky_failures)[:self.NUM_FAILURES_TO_DISPLAY]:
                 self.send_email_for_flaky_failure(flaky_failure)
 
-        self.setProperty('build_summary', message)
+        if RunWebKitTestsInStressMode.FAILURE_MSG_IN_STRESS_MODE not in previous_build_summary:
+            self.setProperty('build_summary', message)
         return SUCCESS
 
     def retry_build(self, message=''):
@@ -5108,6 +5118,9 @@ def hideStepIf(self, results, step):
     def start(self):
         build_summary = self.getProperty('build_summary', 'build successful')
         self.finished(SUCCESS)
+        previous_build_summary = self.getProperty('build_summary', '')
+        if RunWebKitTestsInStressMode.FAILURE_MSG_IN_STRESS_MODE in previous_build_summary:
+            self.build.results = FAILURE
         self.build.buildFinished([build_summary], self.build.results)
         return defer.succeed(None)
 

Tools/CISupport/ews-build/steps_unittest.py

@@ -2243,7 +2243,7 @@ def test_failure(self):
         )
         self.expectOutcome(result=FAILURE, state_string='layout-tests (failure)')
         rc = self.runStep()
-        self.assertEqual(self.getProperty('build_summary'), 'Found test failures')
+        self.assertEqual(self.getProperty('build_summary'), 'Found test failures in stress mode')
         return rc
 
     def test_success(self):
@@ -2346,7 +2346,7 @@ def test_failure(self):
         )
         self.expectOutcome(result=FAILURE, state_string='layout-tests (failure)')
         rc = self.runStep()
-        self.assertEqual(self.getProperty('build_summary'), 'Found test failures')
+        self.assertEqual(self.getProperty('build_summary'), 'Found test failures in stress mode')
         return rc