Skip to content

Commit

Permalink
staticpod: improve operator messages
Browse files Browse the repository at this point in the history
  • Loading branch information
@mfojtik
mfojtik committed Oct 7, 2020
1 parent c803a7b commit 257962a
Show file tree
Hide file tree
Showing 2 changed files with 97 additions and 19 deletions.
31 changes: 25 additions & 6 deletions pkg/operator/staticpod/controller/installer/installer_controller.go
View file
Expand Up @@ -428,6 +428,7 @@ func setAvailableProgressingNodeInstallerFailingConditions(newStatus *operatorv1
numAvailable := 0
numAtLatestRevision := 0
numProgressing := 0
numNodes := len(newStatus.NodeStatuses)
counts := map[int32]int{}
failingCount := map[int32]int{}
failing := map[int32][]string{}
Expand All @@ -452,27 +453,41 @@ func setAvailableProgressingNodeInstallerFailingConditions(newStatus *operatorv1

revisionStrings := []string{}
for _, currentRevision := range Int32KeySet(counts).List() {
count := counts[currentRevision]
revisionStrings = append(revisionStrings, fmt.Sprintf("%d nodes are at revision %d", count, currentRevision))
if count := counts[currentRevision]; count == 1 {
revisionStrings = append(revisionStrings, fmt.Sprintf("1 node achieved revision %d", currentRevision))
} else {
if count == numNodes {
revisionStrings = append(revisionStrings, fmt.Sprintf("all nodes achieved revision %d", currentRevision))
} else {
revisionStrings = append(revisionStrings, fmt.Sprintf("%d nodes achieved revision %d", count, currentRevision))
}
}
}
// if we are progressing and no nodes have achieved that level, we should indicate
if numProgressing > 0 && counts[newStatus.LatestAvailableRevision] == 0 {
revisionStrings = append(revisionStrings, fmt.Sprintf("%d nodes have achieved new revision %d", 0, newStatus.LatestAvailableRevision))
revisionStrings = append(revisionStrings, fmt.Sprintf("none of the nodes have achieved new revision %d", newStatus.LatestAvailableRevision))
}
revisionDescription := strings.Join(revisionStrings, "; ")

if numAvailable > 0 {
message := fmt.Sprintf("%d nodes are active; %s", numAvailable, revisionDescription)
if numAvailable == 1 {
message = fmt.Sprintf("1 node is active; %s", revisionDescription)
}
if numAvailable == numNodes {
message = fmt.Sprintf("all nodes are active; %s", revisionDescription)
}
v1helpers.SetOperatorCondition(&newStatus.Conditions, operatorv1.OperatorCondition{
Type: condition.StaticPodsAvailableConditionType,
Status: operatorv1.ConditionTrue,
Message: fmt.Sprintf("%d nodes are active; %s", numAvailable, revisionDescription),
Message: message,
})
} else {
v1helpers.SetOperatorCondition(&newStatus.Conditions, operatorv1.OperatorCondition{
Type: condition.StaticPodsAvailableConditionType,
Status: operatorv1.ConditionFalse,
Reason: "ZeroNodesActive",
Message: fmt.Sprintf("%d nodes are active; %s", numAvailable, revisionDescription),
Message: strings.Join(append([]string{fmt.Sprintf("none of the nodes are active")}, revisionStrings...), "; "),
})
}

Expand All @@ -496,7 +511,11 @@ func setAvailableProgressingNodeInstallerFailingConditions(newStatus *operatorv1
failingStrings := []string{}
for _, failingRevision := range Int32KeySet(failing).List() {
errorStrings := failing[failingRevision]
failingStrings = append(failingStrings, fmt.Sprintf("%d nodes are failing on revision %d:\n%v", failingCount[failingRevision], failingRevision, strings.Join(errorStrings, "\n")))
if count := failingCount[failingRevision]; count == 1 {
failingStrings = append(failingStrings, fmt.Sprintf("%d/%d node is failing to achieve revision %d:\n%v", failingCount[failingRevision], numNodes, failingRevision, strings.Join(errorStrings, "\n")))
} else {
failingStrings = append(failingStrings, fmt.Sprintf("%d/%d nodes are failing to achieve revision %d:\n%v", failingCount[failingRevision], numNodes, failingRevision, strings.Join(errorStrings, "\n")))
}
}
failingDescription := strings.Join(failingStrings, "; ")

Expand Down
85 changes: 72 additions & 13 deletions pkg/operator/staticpod/controller/installer/installer_controller_test.go
View file
Expand Up @@ -1307,7 +1307,6 @@ func TestNodeToStartRevisionWith(t *testing.T) {
}

func TestSetConditions(t *testing.T) {

type TestCase struct {
name string
latestAvailableRevision int32
Expand All @@ -1316,9 +1315,12 @@ func TestSetConditions(t *testing.T) {
expectedAvailableStatus operatorv1.ConditionStatus
expectedProgressingStatus operatorv1.ConditionStatus
expectedFailingStatus operatorv1.ConditionStatus
expectedAvailableMessage string
expectedPendingMessage string
expectedFailingMessage string
}

testCase := func(name string, available, progressing, failed bool, lastFailedRevision, latest int32, current ...int32) TestCase {
testCase := func(name string, available, progressing, failed bool, lastFailedRevision, latest int32, availableMessage, pendingMessage, failingMessage string, current ...int32) TestCase {
availableStatus := operatorv1.ConditionFalse
pendingStatus := operatorv1.ConditionFalse
expectedFailingStatus := operatorv1.ConditionFalse
Expand All @@ -1331,16 +1333,56 @@ func TestSetConditions(t *testing.T) {
if failed {
expectedFailingStatus = operatorv1.ConditionTrue
}
return TestCase{name, latest, lastFailedRevision, current, availableStatus, pendingStatus, expectedFailingStatus}
return TestCase{
name: name,
latestAvailableRevision: latest,
lastFailedRevision: lastFailedRevision,
currentRevisions: current,
expectedAvailableStatus: availableStatus,
expectedProgressingStatus: pendingStatus,
expectedFailingStatus: expectedFailingStatus,
expectedAvailableMessage: availableMessage,
expectedPendingMessage: pendingMessage,
expectedFailingMessage: failingMessage,
}
}

testCases := []TestCase{
testCase("AvailableProgressingDegraded", true, true, true, 1, 2, 2, 1, 2, 1),
testCase("AvailableProgressing", true, true, false, 0, 2, 2, 1, 2, 1),
testCase("AvailableNotProgressing", true, false, false, 0, 2, 2, 2, 2),
testCase("NotAvailableProgressing", false, true, false, 0, 2, 0, 0),
testCase("NotAvailableAtOldLevelProgressing", true, true, false, 0, 2, 1, 1),
testCase("NotAvailableNotProgressing", false, false, false, 0, 2),
testCase("AvailableProgressingDegraded", true, true, true, 1, 2,
"all nodes are active; 2 nodes achieved revision 1; 2 nodes achieved revision 2",
"2 nodes achieved revision 1; 2 nodes achieved revision 2",
"4/4 nodes are failing to achieve revision 1:\n",
2, 1, 2, 1),
testCase("AvailableProgressingSingular", true, true, true, 1, 2,
"all nodes are active; 3 nodes achieved revision 1; 1 node achieved revision 3; none of the nodes have achieved new revision 2",
"3 nodes achieved revision 1; 1 node achieved revision 3; none of the nodes have achieved new revision 2",
"4/4 nodes are failing to achieve revision 1:\n",
1, 1, 3, 1),
testCase("AvailableProgressing", true, true, false, 0, 2,
"all nodes are active; 2 nodes achieved revision 1; 2 nodes achieved revision 2",
"2 nodes achieved revision 1; 2 nodes achieved revision 2",
"",
2, 1, 2, 1),
testCase("AvailableNotProgressing", true, false, false, 0, 2,
"all nodes are active; all nodes achieved revision 2",
"all nodes achieved revision 2",
"",
2, 2, 2),
testCase("NotAvailableProgressing", false, true, false, 0, 2,
"none of the nodes are active; all nodes achieved revision 0; none of the nodes have achieved new revision 2",
"all nodes achieved revision 0; none of the nodes have achieved new revision 2",
"",
0, 0),
testCase("NotAvailableAtOldLevelProgressing", true, true, false, 0, 2,
"all nodes are active; all nodes achieved revision 1; none of the nodes have achieved new revision 2",
"all nodes achieved revision 1; none of the nodes have achieved new revision 2",
"",
1, 1),
testCase("NotAvailableNotProgressing", false, false, false, 0, 2,
"none of the nodes are active",
"",
"",
),
}

for _, tc := range testCases {
Expand All @@ -1351,28 +1393,45 @@ func TestSetConditions(t *testing.T) {
for _, current := range tc.currentRevisions {
status.NodeStatuses = append(status.NodeStatuses, operatorv1.NodeStatus{CurrentRevision: current, LastFailedRevision: tc.lastFailedRevision})
}
setAvailableProgressingNodeInstallerFailingConditions(status)
if err := setAvailableProgressingNodeInstallerFailingConditions(status); err != nil {
t.Fatalf("unexpected error: %v", err)
}

availableCondition := v1helpers.FindOperatorCondition(status.Conditions, condition.StaticPodsAvailableConditionType)
if availableCondition == nil {
t.Error("Available condition: not found")
} else if availableCondition.Status != tc.expectedAvailableStatus {
return
}
if availableCondition.Status != tc.expectedAvailableStatus {
t.Errorf("Available condition: expected status %v, actual status %v", tc.expectedAvailableStatus, availableCondition.Status)
}
if availableCondition.Message != tc.expectedAvailableMessage {
t.Errorf("expected available message %q, got %q", tc.expectedAvailableMessage, availableCondition.Message)
}

pendingCondition := v1helpers.FindOperatorCondition(status.Conditions, condition.NodeInstallerProgressingConditionType)
if pendingCondition == nil {
t.Error("Progressing condition: not found")
} else if pendingCondition.Status != tc.expectedProgressingStatus {
return
}
if pendingCondition.Status != tc.expectedProgressingStatus {
t.Errorf("Progressing condition: expected status %v, actual status %v", tc.expectedProgressingStatus, pendingCondition.Status)
}
if pendingCondition.Message != tc.expectedPendingMessage {
t.Errorf("expected pending message %q, got %q", tc.expectedPendingMessage, pendingCondition.Message)
}

failingCondition := v1helpers.FindOperatorCondition(status.Conditions, condition.NodeInstallerDegradedConditionType)
if failingCondition == nil {
t.Error("Failing condition: not found")
} else if failingCondition.Status != tc.expectedFailingStatus {
return
}
if failingCondition.Status != tc.expectedFailingStatus {
t.Errorf("Failing condition: expected status %v, actual status %v", tc.expectedFailingStatus, failingCondition.Status)
}
if failingCondition.Message != tc.expectedFailingMessage {
t.Errorf("expected failing message %q, got %q", tc.expectedFailingMessage, failingCondition.Message)
}
})
}

Expand Down

0 comments on commit 257962a

Please sign in to comment.