Skip to content

Commit d7f6bf9

Browse files
Merge pull request #30166 from stbenjam/informing
TRT-2268: Add support for OTE test lifecycles
2 parents 77d41e0 + 1de454b commit d7f6bf9

File tree

5 files changed

+80
-16
lines changed

5 files changed

+80
-16
lines changed

pkg/test/ginkgo/cmd_runsuite.go

Lines changed: 48 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ import (
1919

2020
"github.com/onsi/ginkgo/v2"
2121
"github.com/openshift-eng/openshift-tests-extension/pkg/extension"
22+
"github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests"
2223
configv1 "github.com/openshift/api/config/v1"
2324
"github.com/pkg/errors"
2425
"github.com/sirupsen/logrus"
@@ -289,6 +290,10 @@ func (o *GinkgoRunSuiteOptions) Run(suite *TestSuite, clusterConfig *clusterdisc
289290
return err
290291
}
291292

293+
if len(specs) == 0 {
294+
return fmt.Errorf("no tests to run")
295+
}
296+
292297
tests, err := extensionTestSpecsToOriginTestCases(specs)
293298
if err != nil {
294299
return errors.WithMessage(err, "could not convert test specs to origin test cases")
@@ -563,7 +568,7 @@ func (o *GinkgoRunSuiteOptions) Run(suite *TestSuite, clusterConfig *clusterdisc
563568
}
564569
}
565570

566-
logrus.Warningf("%d tests failed, %d tests permitted to be retried; %d failures are terminal non-retryable failures", len(failing), len(retries), failedUnretriableTestCount)
571+
logrus.Warningf("%d tests failed, %d tests permitted to be retried; %d failures are non-retryable", len(failing), len(retries), failedUnretriableTestCount)
567572

568573
// Run the tests in the retries list.
569574
q := newParallelTestQueue(testRunnerContext)
@@ -708,10 +713,26 @@ func (o *GinkgoRunSuiteOptions) Run(suite *TestSuite, clusterConfig *clusterdisc
708713
wasMasterNodeUpdated = clusterinfo.WasMasterNodeUpdated(events)
709714
}
710715

711-
// report the outcome of the test
712-
if len(failing) > 0 {
713-
names := sets.NewString(testNames(failing)...).List()
714-
fmt.Fprintf(o.Out, "Failing tests:\n\n%s\n\n", strings.Join(names, "\n"))
716+
var blockingFailures, informingFailures []*testCase
717+
for _, test := range failing {
718+
if isBlockingFailure(test) {
719+
blockingFailures = append(blockingFailures, test)
720+
} else {
721+
test.testOutputBytes = []byte(fmt.Sprintf("*** NON-BLOCKING FAILURE: This test failure is not considered terminal because its lifecycle is '%s' and will not prevent the overall suite from passing.\n\n%s",
722+
test.extensionTestResult.Lifecycle,
723+
string(test.testOutputBytes)))
724+
informingFailures = append(informingFailures, test)
725+
}
726+
}
727+
728+
if len(informingFailures) > 0 {
729+
names := sets.NewString(testNames(informingFailures)...).List()
730+
fmt.Fprintf(o.Out, "Informing test failures that don't prevent the overall suite from passing:\n\n\t* %s\n\n", strings.Join(names, "\n\t* "))
731+
}
732+
733+
if len(blockingFailures) > 0 {
734+
names := sets.NewString(testNames(blockingFailures)...).List()
735+
fmt.Fprintf(o.Out, "Blocking test failures:\n\n\t* %s\n\n", strings.Join(names, "\n\t* "))
715736
}
716737

717738
if len(o.JUnitDir) > 0 {
@@ -729,24 +750,35 @@ func (o *GinkgoRunSuiteOptions) Run(suite *TestSuite, clusterConfig *clusterdisc
729750
}
730751
}
731752

732-
if fail > 0 {
733-
if len(failing) > 0 || suite.MaximumAllowedFlakes == 0 {
734-
return fmt.Errorf("%d fail, %d pass, %d skip (%s)", fail, pass, skip, duration)
735-
}
736-
fmt.Fprintf(o.Out, "%d flakes detected, suite allows passing with only flakes\n\n", fail)
737-
}
738-
739-
if syntheticFailure {
753+
switch {
754+
case len(blockingFailures) > 0:
755+
return fmt.Errorf("%d blocking fail, %d informing fail, %d pass, %d skip (%s)", len(blockingFailures), len(informingFailures), pass, skip, duration)
756+
case syntheticFailure:
740757
return fmt.Errorf("failed because an invariant was violated, %d pass, %d skip (%s)\n", pass, skip, duration)
741-
}
742-
if monitorTestResultState != monitor.Succeeded {
758+
case monitorTestResultState != monitor.Succeeded:
743759
return fmt.Errorf("failed due to a MonitorTest failure")
760+
case len(informingFailures) > 0:
761+
fmt.Fprintf(o.Out, "%d informing fail, %d pass, %d skip (%s): suite passes despite failures", len(informingFailures), pass, skip, duration)
762+
default:
763+
fmt.Fprintf(o.Out, "%d pass, %d skip (%s)\n", pass, skip, duration)
744764
}
745765

746-
fmt.Fprintf(o.Out, "%d pass, %d skip (%s)\n", pass, skip, duration)
747766
return ctx.Err()
748767
}
749768

769+
func isBlockingFailure(test *testCase) bool {
770+
if test.extensionTestResult == nil {
771+
return true
772+
}
773+
774+
switch test.extensionTestResult.Lifecycle {
775+
case extensiontests.LifecycleInforming:
776+
return false
777+
default:
778+
return true
779+
}
780+
}
781+
750782
func writeExtensionTestResults(tests []*testCase, dir, filePrefix, fileSuffix string, out io.Writer) error {
751783
// Ensure the directory exists
752784
err := os.MkdirAll(dir, 0755)

pkg/testsuites/standard_suites.go

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -441,6 +441,16 @@ var staticSuites = []ginkgo.TestSuite{
441441
TestTimeout: 120 * time.Minute,
442442
ClusterStabilityDuringTest: ginkgo.Disruptive,
443443
},
444+
{
445+
Name: "openshift/ote",
446+
Description: templates.LongDesc(`
447+
This test suite runs tests to validate OpenShift Test Extension features are working.
448+
`),
449+
Qualifiers: []string{
450+
`name.contains("[OTE]")`,
451+
},
452+
TestTimeout: 1 * time.Minute,
453+
},
444454
}
445455

446456
func withExcludedTestsFilter(baseExpr string) string {

test/extended/extension/extension.go

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
package extension
2+
3+
import (
4+
g "github.com/onsi/ginkgo/v2"
5+
ote "github.com/openshift-eng/openshift-tests-extension/pkg/ginkgo"
6+
e2e "k8s.io/kubernetes/test/e2e/framework"
7+
)
8+
9+
var _ = g.Describe("[sig-ci] [OTE] OpenShift Tests Extension [Suite:openshift/ote]", func() {
10+
defer g.GinkgoRecover()
11+
12+
_ = g.It("should support tests that succeed", func() {})
13+
14+
_ = g.It("should support tests with an informing lifecycle", ote.Informing(), func() {
15+
e2e.Fail("This test is intended to fail.")
16+
})
17+
})

test/extended/include.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ import (
3131
_ "github.com/openshift/origin/test/extended/dns"
3232
_ "github.com/openshift/origin/test/extended/dr"
3333
_ "github.com/openshift/origin/test/extended/etcd"
34+
_ "github.com/openshift/origin/test/extended/extension"
3435
_ "github.com/openshift/origin/test/extended/idling"
3536
_ "github.com/openshift/origin/test/extended/image_ecosystem"
3637
_ "github.com/openshift/origin/test/extended/imageapis"

test/extended/util/annotate/generated/zz_generated.annotations.go

Lines changed: 4 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)