Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Populate node status transition timestamp and reason #4130

Merged
merged 1 commit into from
Feb 7, 2015
Merged

Populate node status transition timestamp and reason #4130

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
45 changes: 37 additions & 8 deletions pkg/cloudprovider/controller/nodecontroller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ package controller

import (
"errors"
"fmt"
"net"
"reflect"
"strings"
Expand Down Expand Up @@ -262,26 +263,43 @@ func (s *NodeController) DoChecks(nodes *api.NodeList) *api.NodeList {
// DoCheck performs health checking for given node.
func (s *NodeController) DoCheck(node *api.Node) []api.NodeCondition {
var conditions []api.NodeCondition

// Check Condition: NodeReady. TODO: More node conditions.
oldReadyCondition := s.getCondition(node, api.NodeReady)
newReadyCondition := s.checkNodeReady(node)
if oldReadyCondition != nil && oldReadyCondition.Status == newReadyCondition.Status {
newReadyCondition.LastTransitionTime = oldReadyCondition.LastTransitionTime
} else {
newReadyCondition.LastTransitionTime = util.Now()
}
conditions = append(conditions, *newReadyCondition)

return conditions
}

// checkNodeReady checks raw node ready condition, without timestamp set.
func (s *NodeController) checkNodeReady(node *api.Node) *api.NodeCondition {
switch status, err := s.kubeletClient.HealthCheck(node.Name); {
case err != nil:
glog.V(2).Infof("NodeController: node %s health check error: %v", node.Name, err)
conditions = append(conditions, api.NodeCondition{
return &api.NodeCondition{
Kind: api.NodeReady,
Status: api.ConditionUnknown,
})
Reason: fmt.Sprintf("Node health check error: %v", err),
}
case status == probe.Failure:
conditions = append(conditions, api.NodeCondition{
return &api.NodeCondition{
Kind: api.NodeReady,
Status: api.ConditionNone,
})
Reason: fmt.Sprintf("Node health check failed: kubelet /healthz endpoint returns not ok"),
}
default:
conditions = append(conditions, api.NodeCondition{
return &api.NodeCondition{
Kind: api.NodeReady,
Status: api.ConditionFull,
})
Reason: fmt.Sprintf("Node health check succeeded: kubelet /healthz endpoint returns ok"),
}
}
glog.V(5).Infof("NodeController: node %q status was %+v", node.Name, conditions)
return conditions
}

// StaticNodes constructs and returns api.NodeList for static nodes. If error
Expand Down Expand Up @@ -340,3 +358,14 @@ func (s *NodeController) canonicalizeName(nodes *api.NodeList) *api.NodeList {
}
return nodes
}

// getCondition returns a condition object for the specific condition
// kind, nil if the condition is not set.
func (s *NodeController) getCondition(node *api.Node, kind api.NodeConditionKind) *api.NodeCondition {
for i := range node.Status.Conditions {
if node.Status.Conditions[i].Kind == kind {
return &node.Status.Conditions[i]
}
}
return nil
}
131 changes: 127 additions & 4 deletions pkg/cloudprovider/controller/nodecontroller_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ import (
"github.com/GoogleCloudPlatform/kubernetes/pkg/client"
fake_cloud "github.com/GoogleCloudPlatform/kubernetes/pkg/cloudprovider/fake"
"github.com/GoogleCloudPlatform/kubernetes/pkg/probe"
"github.com/GoogleCloudPlatform/kubernetes/pkg/util"
)

// FakeNodeHandler is a fake implementation of NodesInterface and NodeInterface.
Expand Down Expand Up @@ -377,6 +378,7 @@ func TestHealthCheckNode(t *testing.T) {
{
Kind: api.NodeReady,
Status: api.ConditionFull,
Reason: "Node health check succeeded: kubelet /healthz endpoint returns ok",
},
},
},
Expand All @@ -390,6 +392,7 @@ func TestHealthCheckNode(t *testing.T) {
{
Kind: api.NodeReady,
Status: api.ConditionNone,
Reason: "Node health check failed: kubelet /healthz endpoint returns not ok",
},
},
},
Expand All @@ -403,6 +406,7 @@ func TestHealthCheckNode(t *testing.T) {
{
Kind: api.NodeReady,
Status: api.ConditionUnknown,
Reason: "Node health check error: Error",
},
},
},
Expand All @@ -411,6 +415,12 @@ func TestHealthCheckNode(t *testing.T) {
for _, item := range table {
nodeController := NewNodeController(nil, "", nil, nil, nil, item.fakeKubeletClient)
conditions := nodeController.DoCheck(item.node)
for i := range conditions {
if conditions[i].LastTransitionTime.IsZero() {
t.Errorf("unexpected zero timestamp")
}
conditions[i].LastTransitionTime = util.Time{}
}
if !reflect.DeepEqual(item.expectedConditions, conditions) {
t.Errorf("expected conditions %+v, got %+v", item.expectedConditions, conditions)
}
Expand Down Expand Up @@ -451,6 +461,98 @@ func TestPopulateNodeIPs(t *testing.T) {
}
}

func TestNodeStatusTransitionTime(t *testing.T) {
table := []struct {
fakeNodeHandler *FakeNodeHandler
fakeKubeletClient *FakeKubeletClient
expectedNodes []*api.Node
expectedRequestCount int
}{
{
fakeNodeHandler: &FakeNodeHandler{
Existing: []*api.Node{
{
ObjectMeta: api.ObjectMeta{Name: "node0"},
Status: api.NodeStatus{
Conditions: []api.NodeCondition{
{
Kind: api.NodeReady,
Status: api.ConditionFull,
Reason: "Node health check succeeded: kubelet /healthz endpoint returns ok",
LastTransitionTime: util.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC),
},
},
},
},
},
},
fakeKubeletClient: &FakeKubeletClient{
Status: probe.Success,
Err: nil,
},
expectedNodes: []*api.Node{},
expectedRequestCount: 1,
},
{
fakeNodeHandler: &FakeNodeHandler{
Existing: []*api.Node{
{
ObjectMeta: api.ObjectMeta{Name: "node0"},
Status: api.NodeStatus{
Conditions: []api.NodeCondition{
{
Kind: api.NodeReady,
Status: api.ConditionFull,
Reason: "Node health check succeeded: kubelet /healthz endpoint returns ok",
LastTransitionTime: util.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC),
},
},
},
},
},
},
fakeKubeletClient: &FakeKubeletClient{
Status: probe.Failure,
Err: nil,
},
expectedNodes: []*api.Node{
{
ObjectMeta: api.ObjectMeta{Name: "node0"},
Status: api.NodeStatus{
Conditions: []api.NodeCondition{
{
Kind: api.NodeReady,
Status: api.ConditionFull,
Reason: "Node health check failed: kubelet /healthz endpoint returns not ok",
LastTransitionTime: util.Now(), // Placeholder expected transition time, due to inability to mock time.
},
},
},
},
},
expectedRequestCount: 2,
},
}

for _, item := range table {
nodeController := NewNodeController(nil, "", []string{"node0"}, nil, item.fakeNodeHandler, item.fakeKubeletClient)
if err := nodeController.SyncNodeStatus(); err != nil {
t.Errorf("unexpected error: %v", err)
}
if item.expectedRequestCount != item.fakeNodeHandler.RequestCount {
t.Errorf("expected %v call, but got %v.", item.expectedRequestCount, item.fakeNodeHandler.RequestCount)
}
for i := range item.fakeNodeHandler.UpdatedNodes {
conditions := item.fakeNodeHandler.UpdatedNodes[i].Status.Conditions
for j := range conditions {
if !conditions[j].LastTransitionTime.After(time.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC)) {
t.Errorf("unexpected timestamp %v", conditions[j].LastTransitionTime)
}
}
}
}
}

func TestSyncNodeStatus(t *testing.T) {
table := []struct {
fakeNodeHandler *FakeNodeHandler
Expand All @@ -474,15 +576,27 @@ func TestSyncNodeStatus(t *testing.T) {
{
ObjectMeta: api.ObjectMeta{Name: "node0"},
Status: api.NodeStatus{
Conditions: []api.NodeCondition{{Kind: api.NodeReady, Status: api.ConditionFull}},
HostIP: "1.2.3.4",
Conditions: []api.NodeCondition{
{
Kind: api.NodeReady,
Status: api.ConditionFull,
Reason: "Node health check succeeded: kubelet /healthz endpoint returns ok",
},
},
HostIP: "1.2.3.4",
},
},
{
ObjectMeta: api.ObjectMeta{Name: "node1"},
Status: api.NodeStatus{
Conditions: []api.NodeCondition{{Kind: api.NodeReady, Status: api.ConditionFull}},
HostIP: "1.2.3.4",
Conditions: []api.NodeCondition{
{
Kind: api.NodeReady,
Status: api.ConditionFull,
Reason: "Node health check succeeded: kubelet /healthz endpoint returns ok",
},
},
HostIP: "1.2.3.4",
},
},
},
Expand All @@ -498,6 +612,15 @@ func TestSyncNodeStatus(t *testing.T) {
if item.fakeNodeHandler.RequestCount != item.expectedRequestCount {
t.Errorf("expected %v call, but got %v.", item.expectedRequestCount, item.fakeNodeHandler.RequestCount)
}
for i := range item.fakeNodeHandler.UpdatedNodes {
conditions := item.fakeNodeHandler.UpdatedNodes[i].Status.Conditions
for j := range conditions {
if conditions[j].LastTransitionTime.IsZero() {
t.Errorf("unexpected zero timestamp")
}
conditions[j].LastTransitionTime = util.Time{}
}
}
if !reflect.DeepEqual(item.expectedNodes, item.fakeNodeHandler.UpdatedNodes) {
t.Errorf("expected nodes %+v, got %+v", item.expectedNodes[0], item.fakeNodeHandler.UpdatedNodes[0])
}
Expand Down