feat: add constructed event for kube api startup window

during single node deployments knowing the window between graceful shutdown end and readyz start is helpful to determine when connection refused events are expected. Signed-off-by: ehila <ehila@redhat.com>
openshift · Apr 19, 2024 · cdbed6f · cdbed6f
1 parent 34cefb1
commit cdbed6f
Show file tree

Hide file tree

Showing 5 changed files with 45 additions and 8 deletions.
diff --git a/e2echart/e2e-chart-template.html b/e2echart/e2e-chart-template.html
@@ -135,7 +135,7 @@ <h5 class="modal-title">Resource</h5>
     }
 
     function isGracefulShutdownActivity(eventInterval) {
-        return (eventInterval.tempSource === "APIServerGracefulShutdown")
+        return (eventInterval.tempSource === "APIServerGracefulShutdown" || eventInterval.tempSource === "APIServerStartup")
     }
 
     function isEndpointConnectivity(eventInterval) {
@@ -260,6 +260,10 @@ <h5 class="modal-title">Resource</h5>
     }
 
     function apiserverShutdownValue(item) {
+        let criticalIndex = item.message.indexOf("startup-analyzer")
+        if (criticalIndex != -1) {
+            return [item.locator, "", "StartupInterval"]
+        }
         // TODO: isolate DNS error into CIClusterDisruption
         return [buildLocatorDisplayString(item.tempStructuredLocator), "", "GracefulShutdownInterval"]
     }
@@ -534,7 +538,7 @@ <h5 class="modal-title">Resource</h5>
                 'Update', 'Drain', 'Reboot', 'OperatingSystemUpdate', 'NodeNotReady', // nodes
                 'Passed', 'Skipped', 'Flaked', 'Failed',  // tests
                 'PodCreated', 'PodScheduled', 'PodTerminating','ContainerWait', 'ContainerStart', 'ContainerNotReady', 'ContainerReady', 'ContainerReadinessFailed', 'ContainerReadinessErrored',  'StartupProbeFailed', // pods
-                'CIClusterDisruption', 'Disruption', // disruption
+                'CIClusterDisruption', 'Disruption', 'GracefulShutdownInterval', 'StartupInterval', // disruption
                 'Degraded', 'Upgradeable', 'False', 'Unknown',
                 'PodLogInfo', 'PodLogWarning', 'PodLogError',
                 'EtcdOther', 'EtcdLeaderFound', 'EtcdLeaderLost', 'EtcdLeaderElected', 'EtcdLeaderMissing'])
@@ -545,7 +549,7 @@ <h5 class="modal-title">Resource</h5>
                 '#1e7bd9', '#4294e6', '#6aaef2', '#96cbff', '#fada5e', // nodes
                 '#3cb043', '#ceba76', '#ffa500', '#d0312d', // tests
                 '#96cbff', '#1e7bd9', '#ffa500', '#ca8dfd', '#9300ff', '#fada5e','#3cb043', '#d0312d', '#d0312d', '#c90076', // pods
-                '#96cbff', '#d0312d', // disruption
+                '#96cbff', '#d0312d', '#6a6b6b','#373838', // disruption
                 '#b65049', '#32b8b6', '#ffffff', '#bbbbbb',
                 '#96cbff', '#fada5e', '#d0312d',
                 '#d3d3de', '#03fc62', '#fc0303', '#fada5e', '#8c5efa']); // EtcdLeadership

diff --git a/pkg/monitor/monitorapi/types.go b/pkg/monitor/monitorapi/types.go
@@ -159,6 +159,7 @@ const (
 	DisruptionSamplerOutageBeganEventReason IntervalReason = "DisruptionSamplerOutageBegan"
 	GracefulAPIServerShutdown               IntervalReason = "GracefulAPIServerShutdown"
 	IncompleteAPIServerShutdown             IntervalReason = "IncompleteAPIServerShutdown"
+	APIServerStartupWindow                  IntervalReason = "StartupWindow"
 
 	HttpClientConnectionLost IntervalReason = "HttpClientConnectionLost"
 
@@ -269,6 +270,7 @@ type IntervalSource string
 const (
 	SourceAlert                     IntervalSource = "Alert"
 	SourceAPIServerShutdown         IntervalSource = "APIServerShutdown"
+	SourceAPIServerStartup          IntervalSource = "APIServerStartup"
 	SourceDisruption                IntervalSource = "Disruption"
 	SourceE2ETest                   IntervalSource = "E2ETest"
 	SourceKubeEvent                 IntervalSource = "KubeEvent"

diff --git a/pkg/monitortestlibrary/pathologicaleventlibrary/duplicated_event_patterns.go b/pkg/monitortestlibrary/pathologicaleventlibrary/duplicated_event_patterns.go
@@ -901,10 +901,13 @@ func newSingleNodeConnectionRefusedEventMatcher(finalIntervals monitorapi.Interv
 	)
 	snoTopology := v1.SingleReplicaTopologyMode
 	ocpAPISeverTargetDownIntervals := finalIntervals.Filter(func(eventInterval monitorapi.Interval) bool {
-		return eventInterval.Source == monitorapi.SourceAlert &&
+		isTargetDown := eventInterval.Source == monitorapi.SourceAlert &&
 			eventInterval.StructuredLocator.Keys[monitorapi.LocatorAlertKey] == "TargetDown" &&
 			(eventInterval.StructuredLocator.Keys[monitorapi.LocatorNamespaceKey] == ocpAPINamespace ||
 				eventInterval.StructuredLocator.Keys[monitorapi.LocatorNamespaceKey] == ocpOAuthAPINamespace)
+
+		isKubeAPIStartupWindow := eventInterval.Source == monitorapi.SourceAPIServerStartup
+		return isTargetDown || isKubeAPIStartupWindow
 	})
 	if len(ocpAPISeverTargetDownIntervals) > 0 {
 		logrus.Infof("found %d OCP APIServer TargetDown intervals", len(ocpAPISeverTargetDownIntervals))

diff --git a/pkg/monitortests/kubeapiserver/apiservergracefulrestart/monitortest.go b/pkg/monitortests/kubeapiserver/apiservergracefulrestart/monitortest.go
@@ -3,6 +3,7 @@ package apiservergracefulrestart
 import (
 	"context"
 	"fmt"
+	"strings"
 	"time"
 
 	"github.com/openshift/origin/pkg/monitortestframework"
@@ -86,6 +87,10 @@ func (*apiserverGracefulShutdownAnalyzer) ConstructComputedIntervals(ctx context
 				delete(startedIntervals, key)
 			}
 
+			if strings.Contains(podRef.Name, "kube-apiserver") {
+				startedIntervals[key] = currInterval.To
+			}
+
 			computedIntervals = append(computedIntervals,
 				monitorapi.NewInterval(monitorapi.APIServerGracefulShutdown, monitorapi.Info).
 					Locator(monitorapi.NewLocator().
@@ -98,6 +103,25 @@ func (*apiserverGracefulShutdownAnalyzer) ConstructComputedIntervals(ctx context
 					Display().
 					Build(startTime, currInterval.To),
 			)
+		case "HasBeenReady":
+			startTime := beginning
+			if prevStart, ok := startedIntervals[key]; ok {
+				startTime = prevStart
+				delete(startedIntervals, key)
+			}
+
+			computedIntervals = append(computedIntervals,
+				monitorapi.NewInterval(monitorapi.SourceAPIServerStartup, monitorapi.Info).
+					Locator(monitorapi.NewLocator().
+						LocateServer(namespaceToServer[podRef.Namespace], nodeName, podRef.Namespace, podRef.Name, true),
+					).
+					Message(monitorapi.NewMessage().
+						Constructed("startup-analyzer").
+						Reason(monitorapi.APIServerStartupWindow),
+					).
+					Display().
+					Build(startTime, currInterval.To),
+			)
 		}
 	}
 
@@ -161,7 +185,7 @@ func interesting(interval monitorapi.Interval) (monitorapi.IntervalReason, bool)
 	reason := interval.StructuredMessage.Reason
 	switch reason {
 	// openshift-apiserver still is using the old event name TerminationStart
-	case "ShutdownInitiated", "TerminationStart", "TerminationGracefulTerminationFinished":
+	case "ShutdownInitiated", "TerminationStart", "TerminationGracefulTerminationFinished", "HasBeenReady":
 		return reason, true
 	default:
 		return "", false

diff --git a/test/extended/testdata/bindata.go b/test/extended/testdata/bindata.go