DO NOT MERGE: PRPQR test for excepted_failures

pkg/monitor/monitorapi/types.go

@@ -200,6 +200,12 @@ const (
 	FailedToDeleteCGroupsPath             IntervalReason = "FailedToDeleteCGroupsPath"
 	FailedToAuthenticateWithOpenShiftUser IntervalReason = "FailedToAuthenticateWithOpenShiftUser"
 	FailedContactingAPIReason             IntervalReason = "FailedContactingAPI"
+
+	UpgradeStartedReason  IntervalReason = "UpgradeStarted"
+	UpgradeVersionReason  IntervalReason = "UpgradeVersion"
+	UpgradeRollbackReason IntervalReason = "UpgradeRollback"
+	UpgradeFailedReason   IntervalReason = "UpgradeFailed"
+	UpgradeCompleteReason IntervalReason = "UpgradeComplete"
 )
 
 type AnnotationKey string
@@ -302,6 +308,10 @@ type Interval struct {
 	To   time.Time
 }
 
+func (r IntervalReason) String() string {
+	return string(r)
+}
+
 func (i Interval) String() string {
 	if i.From.Equal(i.To) {
 		return fmt.Sprintf("%s.%03d %s %s %s",

pkg/monitortestlibrary/platformidentification/upgrade.go

@@ -13,8 +13,8 @@ func DidUpgradeHappenDuringCollection(intervals monitorapi.Intervals, beginning,
 		if event.Source != monitorapi.SourceKubeEvent || event.Locator.Keys[monitorapi.LocatorClusterVersionKey] != "cluster" {
 			continue
 		}
-		reason := string(event.Message.Reason)
-		if reason == "UpgradeStarted" || reason == "UpgradeRollback" {
+		reason := event.Message.Reason
+		if reason == monitorapi.UpgradeStartedReason || reason == monitorapi.UpgradeRollbackReason {
 			return true
 		}
 	}

pkg/monitortests/clusterversionoperator/legacycvomonitortests/monitortest.go

@@ -42,7 +42,7 @@ func (w *legacyMonitorTests) EvaluateTestsFromConstructedIntervals(ctx context.C
 	if isUpgrade {
 		junits = append(junits, testUpgradeOperatorStateTransitions(finalIntervals, w.adminRESTConfig)...)
 	} else {
-		junits = append(junits, testStableSystemOperatorStateTransitions(finalIntervals)...)
+		junits = append(junits, testStableSystemOperatorStateTransitions(finalIntervals, w.adminRESTConfig)...)
 	}
 
 	return junits, nil

pkg/monitortests/clusterversionoperator/legacycvomonitortests/operators.go

@@ -10,21 +10,24 @@ import (
 	"k8s.io/client-go/kubernetes"
 
 	"github.com/openshift/origin/pkg/monitortests/clusterversionoperator/operatorstateanalyzer"
+	"github.com/sirupsen/logrus"
 
 	configv1 "github.com/openshift/api/config/v1"
+	clientconfigv1 "github.com/openshift/client-go/config/clientset/versioned"
 	"github.com/openshift/origin/pkg/monitor/monitorapi"
 	"github.com/openshift/origin/pkg/monitortestlibrary/platformidentification"
 	platformidentification2 "github.com/openshift/origin/pkg/monitortestlibrary/platformidentification"
 	"github.com/openshift/origin/pkg/test/ginkgo/junitapi"
+	exutil "github.com/openshift/origin/test/extended/util"
 	"k8s.io/client-go/rest"
 )
 
 // exceptionCallback consumes a suspicious condition and returns an
 // exception string if does not think the condition should be fatal.
-type exceptionCallback func(operator string, condition *configv1.ClusterOperatorStatusCondition, clientConfig *rest.Config) (string, error)
+type exceptionCallback func(operator string, condition *configv1.ClusterOperatorStatusCondition, eventInterval monitorapi.Interval, clientConfig *rest.Config) (string, error)
 
-func testStableSystemOperatorStateTransitions(events monitorapi.Intervals) []*junitapi.JUnitTestCase {
-	except := func(_ string, condition *configv1.ClusterOperatorStatusCondition, _ *rest.Config) (string, error) {
+func testStableSystemOperatorStateTransitions(events monitorapi.Intervals, clientConfig *rest.Config) []*junitapi.JUnitTestCase {
+	except := func(_ string, condition *configv1.ClusterOperatorStatusCondition, _ monitorapi.Interval, clientConfig *rest.Config) (string, error) {
 		if condition.Status == configv1.ConditionTrue {
 			if condition.Type == configv1.OperatorAvailable {
 				return fmt.Sprintf("%s=%s is the happy case", condition.Type, condition.Status), nil
@@ -35,14 +38,118 @@ func testStableSystemOperatorStateTransitions(events monitorapi.Intervals) []*ju
 			}
 		}
 
-		return "We are not worried about Available=False or Degraded=True blips for stable-system tests yet.", nil
+		isSingleNode, err := isSingleNodeCheck(clientConfig)
+		if err != nil {
+			logrus.Warnf("Error checking for Single Node configuration on stable system (unable to make exception): %v", err)
+			isSingleNode = false
+		}
+
+		// For the non-upgrade case, if any operator has Available=False, fail the test.
+		if condition.Type == configv1.OperatorAvailable {
+
+			// We'll add an exception for single node for now.
+			if condition.Status == configv1.ConditionFalse && !isSingleNode {
+				return "", nil
+			}
+		}
+		return "We are not worried about Degraded=True blips for stable-system tests yet.", nil
+	}
+
+	return testOperatorStateTransitions(events, []configv1.ClusterStatusConditionType{configv1.OperatorAvailable, configv1.OperatorDegraded}, except, clientConfig)
+}
+
+func isSingleNodeCheck(clientConfig *rest.Config) (bool, error) {
+	configClient, err := clientconfigv1.NewForConfig(clientConfig)
+	if err != nil {
+		logrus.Warnf("Error creating config client to check for Single Node configuration: %v", err)
+		return false, err
+	}
+	return exutil.IsSingleNode(context.Background(), configClient)
+}
+
+// isInUpgradeWindow determines if the given eventInterval falls within an upgrade window.
+// UpgradeStart and UpgradeRollback events start upgrade windows and can end and already started upgrade window.
+// UpgradeComplete and UpgradeFailed events end upgrade windows; if there was not an already started upgrade window,
+// we ignore the event.
+// If we don't find any upgrade ending point, we assume the ending point is at the end of the test.
+func isInUpgradeWindow(eventList monitorapi.Intervals, eventInterval monitorapi.Interval) bool {
+	type upgradeWindowHolder struct {
+		startInterval *monitorapi.Interval
+		endInterval   *monitorapi.Interval
+	}
+
+	var upgradeWindows []*upgradeWindowHolder
+	var currentWindow *upgradeWindowHolder
+
+	for _, event := range eventList {
+		if event.Source != monitorapi.SourceKubeEvent || event.Locator.Keys[monitorapi.LocatorClusterVersionKey] != "cluster" {
+			continue
+		}
+
+		switch event.Message.Reason {
+		case monitorapi.UpgradeStartedReason, monitorapi.UpgradeRollbackReason:
+			if currentWindow != nil {
+				// Close current window since there's already an upgrade window started
+				currentWindow.endInterval = &monitorapi.Interval{
+					Condition: monitorapi.Condition{
+						Message: monitorapi.Message{
+							Reason: event.Message.Reason,
+						},
+					},
+					From: event.From,
+					To:   event.To,
+				}
+			}
+
+			// Start new window
+			currentWindow = &upgradeWindowHolder{
+				startInterval: &monitorapi.Interval{
+					Condition: monitorapi.Condition{
+						Message: monitorapi.Message{
+							Reason: event.Message.Reason,
+						},
+					},
+					From: event.From,
+					To:   event.To,
+				},
+			}
+			upgradeWindows = append(upgradeWindows, currentWindow)
+		case monitorapi.UpgradeCompleteReason, monitorapi.UpgradeFailedReason:
+			if currentWindow != nil {
+				if currentWindow.endInterval == nil {
+					// End current window
+					currentWindow.endInterval = &monitorapi.Interval{
+						Condition: monitorapi.Condition{
+							Message: monitorapi.Message{
+								Reason: event.Message.Reason,
+							},
+						},
+						From: event.From,
+						To:   event.To,
+					}
+				}
+			} else {
+				// We have no current window which means that the events indicate we completed
+				// or failed an upgrade without starting one.  This is stange situation that
+				// we should not see; in this case, there is no upgrade window to check against.
+				logrus.Warnf("Found upgrade completion or failed event without a start or rollback event: %v", event)
+			}
+		}
 	}
 
-	return testOperatorStateTransitions(events, []configv1.ClusterStatusConditionType{configv1.OperatorAvailable, configv1.OperatorDegraded}, except)
+	for _, upgradeWindow := range upgradeWindows {
+		if eventInterval.From.After(upgradeWindow.startInterval.From) {
+			if upgradeWindow.endInterval == nil || eventInterval.To.Before(upgradeWindow.endInterval.To) {
+				return true
+			}
+		}
+	}
+
+	return false
 }
 
 func testUpgradeOperatorStateTransitions(events monitorapi.Intervals, clientConfig *rest.Config) []*junitapi.JUnitTestCase {
-	except := func(operator string, condition *configv1.ClusterOperatorStatusCondition, clientConfig *rest.Config) (string, error) {
+	except := func(operator string, condition *configv1.ClusterOperatorStatusCondition, eventInterval monitorapi.Interval, clientConfig *rest.Config) (string, error) {
 		if condition.Status == configv1.ConditionTrue {
 			if condition.Type == configv1.OperatorAvailable {
 				return fmt.Sprintf("%s=%s is the happy case", condition.Type, condition.Status), nil
@@ -57,6 +164,22 @@ func testUpgradeOperatorStateTransitions(events monitorapi.Intervals, clientConf
 			return "We are not worried about Degraded=True blips for update tests yet.", nil
 		}
 
+		var availableEqualsFalseAllowed bool
+		if condition.Type == configv1.OperatorAvailable && condition.Status == configv1.ConditionFalse {
+			availableEqualsFalseAllowed = isInUpgradeWindow(events, eventInterval) && eventInterval.To.Sub(eventInterval.From) < 10*time.Minute
+		}
+
+		isSingleNode, err := isSingleNodeCheck(clientConfig)
+		if err != nil {
+			logrus.Warnf("Error checking for Single Node configuration on upgrade (unable to make exception): %v", err)
+			isSingleNode = false
+		}
+
+		// We'll add an exception for single node for now.
+		if !availableEqualsFalseAllowed && !isSingleNode {
+			return "", nil
+		}
+
 		switch operator {
 		case "authentication":
 			if condition.Type == configv1.OperatorAvailable && condition.Status == configv1.ConditionFalse && (condition.Reason == "APIServices_Error" || condition.Reason == "APIServerDeployment_NoDeployment" || condition.Reason == "APIServerDeployment_NoPod" || condition.Reason == "APIServerDeployment_PreconditionNotFulfilled" || condition.Reason == "APIServices_PreconditionNotReady" || condition.Reason == "OAuthServerDeployment_NoDeployment" || condition.Reason == "OAuthServerRouteEndpointAccessibleController_EndpointUnavailable" || condition.Reason == "OAuthServerServiceEndpointAccessibleController_EndpointUnavailable" || condition.Reason == "WellKnown_NotReady") {
@@ -99,7 +222,7 @@ func testUpgradeOperatorStateTransitions(events monitorapi.Intervals, clientConf
 			}
 		case "image-registry":
 			if replicaCount, _ := checkReplicas("openshift-image-registry", operator, clientConfig); replicaCount == 1 {
-				return "image-registry has only single replica", nil
+				return "https://issues.redhat.com/browse/OCPBUGS-22382", nil
 			}
 		}
 
@@ -128,7 +251,7 @@ func checkReplicas(namespace string, operator string, clientConfig *rest.Config)
 	return 0, fmt.Errorf("Error fetching replicas")
 }
 
-func testOperatorStateTransitions(events monitorapi.Intervals, conditionTypes []configv1.ClusterStatusConditionType, except exceptionCallback, clientConfig ...*rest.Config) []*junitapi.JUnitTestCase {
+func testOperatorStateTransitions(events monitorapi.Intervals, conditionTypes []configv1.ClusterStatusConditionType, except exceptionCallback, clientConfig *rest.Config) []*junitapi.JUnitTestCase {
 	ret := []*junitapi.JUnitTestCase{}
 
 	var start, stop time.Time
@@ -195,11 +318,7 @@ func testOperatorStateTransitions(events monitorapi.Intervals, conditionTypes []
 				if len(concurrentE2E) > 0 {
 					failure = fmt.Sprintf("%s\n%d tests failed during this blip (%v to %v): %v", failure, len(concurrentE2E), eventInterval.From, eventInterval.From, strings.Join(concurrentE2E, "\n"))
 				}
-				var Config *rest.Config
-				if len(clientConfig) > 0 {
-					Config = clientConfig[0]
-				}
-				exception, err := except(operatorName, condition, Config)
+				exception, err := except(operatorName, condition, eventInterval, clientConfig)
 				if err != nil || exception == "" {
 					fatal = append(fatal, failure)
 				} else {