Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Delete node lease if the corresponding node is deleted #70034

Merged
merged 1 commit into from
Nov 10, 2018
Merged

Delete node lease if the corresponding node is deleted #70034

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
3 changes: 3 additions & 0 deletions pkg/kubelet/nodelease/BUILD
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,10 +25,13 @@ go_test(
embed = [":go_default_library"],
deps = [
"//staging/src/k8s.io/api/coordination/v1beta1:go_default_library",
"//staging/src/k8s.io/api/core/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/api/equality:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/types:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/clock:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/diff:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library",
"//vendor/k8s.io/utils/pointer:go_default_library",
],
)
Expand Down
54 changes: 41 additions & 13 deletions pkg/kubelet/nodelease/controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,8 @@ type Controller interface {
}

type controller struct {
client coordclientset.LeaseInterface
client clientset.Interface
leaseClient coordclientset.LeaseInterface
holderIdentity string
leaseDurationSeconds int32
renewInterval time.Duration
Expand All @@ -67,7 +68,8 @@ func NewController(clock clock.Clock, client clientset.Interface, holderIdentity
leaseClient = client.CoordinationV1beta1().Leases(corev1.NamespaceNodeLease)
}
return &controller{
client: leaseClient,
client: client,
leaseClient: leaseClient,
holderIdentity: holderIdentity,
leaseDurationSeconds: leaseDurationSeconds,
renewInterval: renewInterval,
Expand All @@ -78,8 +80,8 @@ func NewController(clock clock.Clock, client clientset.Interface, holderIdentity

// Run runs the controller
func (c *controller) Run(stopCh <-chan struct{}) {
if c.client == nil {
glog.Infof("node lease controller has nil client, will not claim or renew leases")
if c.leaseClient == nil {
glog.Infof("node lease controller has nil lease client, will not claim or renew leases")
return
}
wait.Until(c.sync, c.renewInterval, stopCh)
Expand Down Expand Up @@ -120,10 +122,10 @@ func (c *controller) backoffEnsureLease() (*coordv1beta1.Lease, bool) {
// ensureLease creates the lease if it does not exist. Returns the lease and
// a bool (true if this call created the lease), or any error that occurs.
func (c *controller) ensureLease() (*coordv1beta1.Lease, bool, error) {
lease, err := c.client.Get(c.holderIdentity, metav1.GetOptions{})
lease, err := c.leaseClient.Get(c.holderIdentity, metav1.GetOptions{})
if apierrors.IsNotFound(err) {
// lease does not exist, create it
lease, err := c.client.Create(c.newLease(nil))
lease, err := c.leaseClient.Create(c.newLease(nil))
if err != nil {
return nil, false, err
}
Expand All @@ -140,7 +142,7 @@ func (c *controller) ensureLease() (*coordv1beta1.Lease, bool, error) {
// call this once you're sure the lease has been created
func (c *controller) retryUpdateLease(base *coordv1beta1.Lease) {
for i := 0; i < maxUpdateRetries; i++ {
_, err := c.client.Update(c.newLease(base))
_, err := c.leaseClient.Update(c.newLease(base))
if err == nil {
return
}
Expand All @@ -155,18 +157,44 @@ func (c *controller) retryUpdateLease(base *coordv1beta1.Lease) {
// newLease constructs a new lease if base is nil, or returns a copy of base
// with desired state asserted on the copy.
func (c *controller) newLease(base *coordv1beta1.Lease) *coordv1beta1.Lease {
// Use the bare minimum set of fields; other fields exist for debugging/legacy,
// but we don't need to make node heartbeats more complicated by using them.
var lease *coordv1beta1.Lease
if base == nil {
lease = &coordv1beta1.Lease{}
lease = &coordv1beta1.Lease{
ObjectMeta: metav1.ObjectMeta{
Name: c.holderIdentity,
Namespace: corev1.NamespaceNodeLease,
},
Spec: coordv1beta1.LeaseSpec{
HolderIdentity: pointer.StringPtr(c.holderIdentity),
LeaseDurationSeconds: pointer.Int32Ptr(c.leaseDurationSeconds),
},
}
} else {
lease = base.DeepCopy()
}
// Use the bare minimum set of fields; other fields exist for debugging/legacy,
// but we don't need to make node heartbeats more complicated by using them.
lease.Name = c.holderIdentity
lease.Spec.HolderIdentity = pointer.StringPtr(c.holderIdentity)
lease.Spec.LeaseDurationSeconds = pointer.Int32Ptr(c.leaseDurationSeconds)
lease.Spec.RenewTime = &metav1.MicroTime{Time: c.clock.Now()}

// Setting owner reference needs node's UID. Note that it is different from
// kubelet.nodeRef.UID. When lease is initially created, it is possible that
// the connection between master and node is not ready yet. So try to set
// owner reference every time when renewing the lease, until successful.
if lease.OwnerReferences == nil || len(lease.OwnerReferences) == 0 {
if node, err := c.client.CoreV1().Nodes().Get(c.holderIdentity, metav1.GetOptions{}); err == nil {
lease.OwnerReferences = []metav1.OwnerReference{
{
APIVersion: corev1.SchemeGroupVersion.WithKind("Node").Version,
Kind: corev1.SchemeGroupVersion.WithKind("Node").Kind,
Name: c.holderIdentity,
UID: node.UID,
},
}
} else {
glog.Errorf("failed to get node %q when trying to set owner ref to the node lease: %v", c.holderIdentity, err)
}
}

return lease
}

Expand Down
118 changes: 108 additions & 10 deletions pkg/kubelet/nodelease/controller_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,63 +21,161 @@ import (
"time"

coordv1beta1 "k8s.io/api/coordination/v1beta1"
corev1 "k8s.io/api/core/v1"
apiequality "k8s.io/apimachinery/pkg/api/equality"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/clock"
"k8s.io/apimachinery/pkg/util/diff"
"k8s.io/client-go/kubernetes/fake"
"k8s.io/utils/pointer"
)

func TestNewLease(t *testing.T) {
fakeClock := clock.NewFakeClock(time.Now())
node := &corev1.Node{
ObjectMeta: metav1.ObjectMeta{
Name: "foo",
UID: types.UID("foo-uid"),
},
}
cases := []struct {
desc string
controller *controller
base *coordv1beta1.Lease
expect *coordv1beta1.Lease
}{
{
desc: "nil base",
desc: "nil base without node",
controller: &controller{
client: fake.NewSimpleClientset(),
holderIdentity: node.Name,
leaseDurationSeconds: 10,
clock: fakeClock,
},
base: nil,
expect: &coordv1beta1.Lease{
ObjectMeta: metav1.ObjectMeta{
Name: node.Name,
Namespace: corev1.NamespaceNodeLease,
},
Spec: coordv1beta1.LeaseSpec{
HolderIdentity: pointer.StringPtr(node.Name),
LeaseDurationSeconds: pointer.Int32Ptr(10),
RenewTime: &metav1.MicroTime{Time: fakeClock.Now()},
},
},
},
{
desc: "nil base with node",
controller: &controller{
holderIdentity: "foo",
client: fake.NewSimpleClientset(node),
holderIdentity: node.Name,
leaseDurationSeconds: 10,
clock: fakeClock,
},
base: nil,
expect: &coordv1beta1.Lease{
ObjectMeta: metav1.ObjectMeta{
Name: "foo",
Name: node.Name,
Namespace: corev1.NamespaceNodeLease,
OwnerReferences: []metav1.OwnerReference{
{
APIVersion: corev1.SchemeGroupVersion.WithKind("Node").Version,
Kind: corev1.SchemeGroupVersion.WithKind("Node").Kind,
Name: node.Name,
UID: node.UID,
},
},
},
Spec: coordv1beta1.LeaseSpec{
HolderIdentity: pointer.StringPtr(node.Name),
LeaseDurationSeconds: pointer.Int32Ptr(10),
RenewTime: &metav1.MicroTime{Time: fakeClock.Now()},
},
},
},
{
desc: "non-nil base without owner ref, renew time is updated",
controller: &controller{
client: fake.NewSimpleClientset(node),
holderIdentity: node.Name,
leaseDurationSeconds: 10,
clock: fakeClock,
},
base: &coordv1beta1.Lease{
ObjectMeta: metav1.ObjectMeta{
Name: node.Name,
Namespace: corev1.NamespaceNodeLease,
},
Spec: coordv1beta1.LeaseSpec{
HolderIdentity: pointer.StringPtr(node.Name),
LeaseDurationSeconds: pointer.Int32Ptr(10),
RenewTime: &metav1.MicroTime{Time: fakeClock.Now().Add(-10 * time.Second)},
},
},
expect: &coordv1beta1.Lease{
ObjectMeta: metav1.ObjectMeta{
Name: node.Name,
Namespace: corev1.NamespaceNodeLease,
OwnerReferences: []metav1.OwnerReference{
{
APIVersion: corev1.SchemeGroupVersion.WithKind("Node").Version,
Kind: corev1.SchemeGroupVersion.WithKind("Node").Kind,
Name: node.Name,
UID: node.UID,
},
},
},
Spec: coordv1beta1.LeaseSpec{
HolderIdentity: pointer.StringPtr("foo"),
HolderIdentity: pointer.StringPtr(node.Name),
LeaseDurationSeconds: pointer.Int32Ptr(10),
RenewTime: &metav1.MicroTime{Time: fakeClock.Now()},
},
},
},
{
desc: "non-nil base renew time is updated",
desc: "non-nil base with owner ref, renew time is updated",
controller: &controller{
holderIdentity: "foo",
client: fake.NewSimpleClientset(node),
holderIdentity: node.Name,
leaseDurationSeconds: 10,
clock: fakeClock,
},
base: &coordv1beta1.Lease{
ObjectMeta: metav1.ObjectMeta{
Name: "foo",
Name: node.Name,
Namespace: corev1.NamespaceNodeLease,
OwnerReferences: []metav1.OwnerReference{
{
APIVersion: corev1.SchemeGroupVersion.WithKind("Node").Version,
Kind: corev1.SchemeGroupVersion.WithKind("Node").Kind,
Name: node.Name,
UID: node.UID,
},
},
},
Spec: coordv1beta1.LeaseSpec{
HolderIdentity: pointer.StringPtr("foo"),
HolderIdentity: pointer.StringPtr(node.Name),
LeaseDurationSeconds: pointer.Int32Ptr(10),
RenewTime: &metav1.MicroTime{Time: fakeClock.Now().Add(-10 * time.Second)},
},
},
expect: &coordv1beta1.Lease{
ObjectMeta: metav1.ObjectMeta{
Name: "foo",
Name: node.Name,
Namespace: corev1.NamespaceNodeLease,
OwnerReferences: []metav1.OwnerReference{
{
APIVersion: corev1.SchemeGroupVersion.WithKind("Node").Version,
Kind: corev1.SchemeGroupVersion.WithKind("Node").Kind,
Name: node.Name,
UID: node.UID,
},
},
},
Spec: coordv1beta1.LeaseSpec{
HolderIdentity: pointer.StringPtr("foo"),
HolderIdentity: pointer.StringPtr(node.Name),
LeaseDurationSeconds: pointer.Int32Ptr(10),
RenewTime: &metav1.MicroTime{Time: fakeClock.Now()},
},
Expand Down
1 change: 1 addition & 0 deletions test/e2e/lifecycle/BUILD
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ go_library(
"framework.go",
"ha_master.go",
"kubelet_security.go",
"node_lease.go",
"reboot.go",
"resize_nodes.go",
"restart.go",
Expand Down