@@ -19,6 +19,7 @@ import (
19
19
20
20
"github.com/onsi/ginkgo/v2"
21
21
"github.com/openshift-eng/openshift-tests-extension/pkg/extension"
22
+ "github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests"
22
23
configv1 "github.com/openshift/api/config/v1"
23
24
"github.com/pkg/errors"
24
25
"github.com/sirupsen/logrus"
@@ -289,6 +290,10 @@ func (o *GinkgoRunSuiteOptions) Run(suite *TestSuite, clusterConfig *clusterdisc
289
290
return err
290
291
}
291
292
293
+ if len (specs ) == 0 {
294
+ return fmt .Errorf ("no tests to run" )
295
+ }
296
+
292
297
tests , err := extensionTestSpecsToOriginTestCases (specs )
293
298
if err != nil {
294
299
return errors .WithMessage (err , "could not convert test specs to origin test cases" )
@@ -563,7 +568,7 @@ func (o *GinkgoRunSuiteOptions) Run(suite *TestSuite, clusterConfig *clusterdisc
563
568
}
564
569
}
565
570
566
- logrus .Warningf ("%d tests failed, %d tests permitted to be retried; %d failures are terminal non-retryable failures " , len (failing ), len (retries ), failedUnretriableTestCount )
571
+ logrus .Warningf ("%d tests failed, %d tests permitted to be retried; %d failures are non-retryable" , len (failing ), len (retries ), failedUnretriableTestCount )
567
572
568
573
// Run the tests in the retries list.
569
574
q := newParallelTestQueue (testRunnerContext )
@@ -708,10 +713,26 @@ func (o *GinkgoRunSuiteOptions) Run(suite *TestSuite, clusterConfig *clusterdisc
708
713
wasMasterNodeUpdated = clusterinfo .WasMasterNodeUpdated (events )
709
714
}
710
715
711
- // report the outcome of the test
712
- if len (failing ) > 0 {
713
- names := sets .NewString (testNames (failing )... ).List ()
714
- fmt .Fprintf (o .Out , "Failing tests:\n \n %s\n \n " , strings .Join (names , "\n " ))
716
+ var blockingFailures , informingFailures []* testCase
717
+ for _ , test := range failing {
718
+ if isBlockingFailure (test ) {
719
+ blockingFailures = append (blockingFailures , test )
720
+ } else {
721
+ test .testOutputBytes = []byte (fmt .Sprintf ("*** NON-BLOCKING FAILURE: This test failure is not considered terminal because its lifecycle is '%s' and will not prevent the overall suite from passing.\n \n %s" ,
722
+ test .extensionTestResult .Lifecycle ,
723
+ string (test .testOutputBytes )))
724
+ informingFailures = append (informingFailures , test )
725
+ }
726
+ }
727
+
728
+ if len (informingFailures ) > 0 {
729
+ names := sets .NewString (testNames (informingFailures )... ).List ()
730
+ fmt .Fprintf (o .Out , "Informing test failures that don't prevent the overall suite from passing:\n \n \t * %s\n \n " , strings .Join (names , "\n \t * " ))
731
+ }
732
+
733
+ if len (blockingFailures ) > 0 {
734
+ names := sets .NewString (testNames (blockingFailures )... ).List ()
735
+ fmt .Fprintf (o .Out , "Blocking test failures:\n \n \t * %s\n \n " , strings .Join (names , "\n \t * " ))
715
736
}
716
737
717
738
if len (o .JUnitDir ) > 0 {
@@ -729,24 +750,35 @@ func (o *GinkgoRunSuiteOptions) Run(suite *TestSuite, clusterConfig *clusterdisc
729
750
}
730
751
}
731
752
732
- if fail > 0 {
733
- if len (failing ) > 0 || suite .MaximumAllowedFlakes == 0 {
734
- return fmt .Errorf ("%d fail, %d pass, %d skip (%s)" , fail , pass , skip , duration )
735
- }
736
- fmt .Fprintf (o .Out , "%d flakes detected, suite allows passing with only flakes\n \n " , fail )
737
- }
738
-
739
- if syntheticFailure {
753
+ switch {
754
+ case len (blockingFailures ) > 0 :
755
+ return fmt .Errorf ("%d blocking fail, %d informing fail, %d pass, %d skip (%s)" , len (blockingFailures ), len (informingFailures ), pass , skip , duration )
756
+ case syntheticFailure :
740
757
return fmt .Errorf ("failed because an invariant was violated, %d pass, %d skip (%s)\n " , pass , skip , duration )
741
- }
742
- if monitorTestResultState != monitor .Succeeded {
758
+ case monitorTestResultState != monitor .Succeeded :
743
759
return fmt .Errorf ("failed due to a MonitorTest failure" )
760
+ case len (informingFailures ) > 0 :
761
+ fmt .Fprintf (o .Out , "%d informing fail, %d pass, %d skip (%s): suite passes despite failures" , len (informingFailures ), pass , skip , duration )
762
+ default :
763
+ fmt .Fprintf (o .Out , "%d pass, %d skip (%s)\n " , pass , skip , duration )
744
764
}
745
765
746
- fmt .Fprintf (o .Out , "%d pass, %d skip (%s)\n " , pass , skip , duration )
747
766
return ctx .Err ()
748
767
}
749
768
769
+ func isBlockingFailure (test * testCase ) bool {
770
+ if test .extensionTestResult == nil {
771
+ return true
772
+ }
773
+
774
+ switch test .extensionTestResult .Lifecycle {
775
+ case extensiontests .LifecycleInforming :
776
+ return false
777
+ default :
778
+ return true
779
+ }
780
+ }
781
+
750
782
func writeExtensionTestResults (tests []* testCase , dir , filePrefix , fileSuffix string , out io.Writer ) error {
751
783
// Ensure the directory exists
752
784
err := os .MkdirAll (dir , 0755 )
0 commit comments