Merge pull request #27852 from deads2k/cgroups

add specific test for failing cgroups path

pkg/monitor/intervalcreation/node.go

@@ -172,6 +172,7 @@ func IntervalsFromAuditLogs(ctx context.Context, kubeClient kubernetes.Interface
 // eventsFromKubeletLogs returns the produced intervals.  Any errors during this creation are logged, but
 // not returned because this is a best effort step
 func eventsFromKubeletLogs(nodeName string, kubeletLog []byte) monitorapi.Intervals {
+	nodeLocator := monitorapi.NodeLocator(nodeName)
 	ret := monitorapi.Intervals{}
 
 	scanner := bufio.NewScanner(bytes.NewBuffer(kubeletLog))
@@ -184,6 +185,7 @@ func eventsFromKubeletLogs(nodeName string, kubeletLog []byte) monitorapi.Interv
 		ret = append(ret, kubeletNodeHttpClientConnectionLostError(currLine)...)
 		ret = append(ret, startupProbeError(currLine)...)
 		ret = append(ret, errParsingSignature(currLine)...)
+		ret = append(ret, failedToDeleteCGroupsPath(nodeLocator, currLine)...)
 	}
 
 	return ret
@@ -412,6 +414,26 @@ func statusHttpClientConnectionLostError(logLine string) monitorapi.Intervals {
 	})
 }
 
+func failedToDeleteCGroupsPath(nodeLocator, logLine string) monitorapi.Intervals {
+	if !strings.Contains(logLine, "Failed to delete cgroup paths") {
+		return nil
+	}
+
+	failureTime := kubeletLogTime(logLine)
+
+	return monitorapi.Intervals{
+		{
+			Condition: monitorapi.Condition{
+				Level:   monitorapi.Error,
+				Locator: nodeLocator,
+				Message: monitorapi.ReasonedMessage("FailedToDeleteCGroupsPath", logLine),
+			},
+			From: failureTime,
+			To:   failureTime.Add(1 * time.Second),
+		},
+	}
+}
+
 var nodeRefRegex = regexp.MustCompile(`error getting node \\"(?P<NODEID>[a-z0-9.-]+)\\"`)
 var nodeOutputRegex = regexp.MustCompile(`err="(?P<OUTPUT>.+)"`)
 

pkg/synthetictests/event_junits.go

@@ -73,6 +73,7 @@ func StableSystemEventInvariants(events monitorapi.Intervals, duration time.Dura
 	tests = append(tests, testNodeHasSufficientMemory(events)...)
 	tests = append(tests, testNodeHasSufficientPID(events)...)
 
+	tests = append(tests, testFailedToDeleteCGroupsPath(events)...)
 	tests = append(tests, testHttpConnectionLost(events)...)
 	tests = append(tests, testMarketplaceStartupProbeFailure(events)...)
 	tests = append(tests, testErrImagePullUnrecognizedSignatureFormat(events)...)
@@ -143,6 +144,7 @@ func SystemUpgradeEventInvariants(events monitorapi.Intervals, duration time.Dur
 	tests = append(tests, testNodeHasSufficientMemory(events)...)
 	tests = append(tests, testNodeHasSufficientPID(events)...)
 
+	tests = append(tests, testFailedToDeleteCGroupsPath(events)...)
 	tests = append(tests, testHttpConnectionLost(events)...)
 	tests = append(tests, testMarketplaceStartupProbeFailure(events)...)
 	tests = append(tests, testErrImagePullUnrecognizedSignatureFormat(events)...)

pkg/synthetictests/kubelet.go

@@ -100,6 +100,33 @@ func testHttpConnectionLost(events monitorapi.Intervals) []*junitapi.JUnitTestCa
 	return []*junitapi.JUnitTestCase{failure, success}
 }
 
+func testFailedToDeleteCGroupsPath(events monitorapi.Intervals) []*junitapi.JUnitTestCase {
+	const testName = "[sig-node] kubelet should be able to delete cgroups path"
+
+	var failures []string
+	for _, event := range events {
+		if strings.Contains(event.Message, "reason/FailedToDeleteCGroupsPath") {
+			failures = append(failures, fmt.Sprintf("%v - %v", event.Locator, event.Message))
+		}
+	}
+
+	if len(failures) == 0 {
+		success := &junitapi.JUnitTestCase{Name: testName}
+		return []*junitapi.JUnitTestCase{success}
+	}
+
+	failure := &junitapi.JUnitTestCase{
+		Name:      testName,
+		SystemOut: strings.Join(failures, "\n"),
+		FailureOutput: &junitapi.FailureOutput{
+			Output: fmt.Sprintf("kubelet logs contain %d failures to delete cgroups path.\n\n%v", len(failures), strings.Join(failures, "\n")),
+		},
+	}
+	// add success to flake the test because this fails very commonly.
+	success := &junitapi.JUnitTestCase{Name: testName}
+	return []*junitapi.JUnitTestCase{failure, success}
+}
+
 func testKubeAPIServerGracefulTermination(events monitorapi.Intervals) []*junitapi.JUnitTestCase {
 	const testName = "[sig-api-machinery] kube-apiserver terminates within graceful termination period"