From 74eae467e88f6be5d4e117ca73ff28de385868b6 Mon Sep 17 00:00:00 2001 From: Dan Williams Date: Thu, 8 Jul 2021 13:20:15 -0500 Subject: [PATCH] services: log errors and don't panic if the service doesn't exist anymore If ovnkube-master is a bit slow and the service is already gone from the apiserver and informer cache, we still want to clean it up. But we don't want to pass a nil service to Eventf() which will cause the following panic: I0708 15:46:33.636318 1 services_controller.go:518] Deleting service cluster-density-503fcc1a-1066-44d9-bef7-43292b046b06-151/deployment-2pod-151-1 E0708 15:46:35.468114 1 utils.go:93] Error deleting VIP [172.30.55.216:80] on OVN LoadBalancer [01801284-6384-498c-8ef8-a08a6e63c77f 072075fd-eb15-47cc-bb17-05c8b4b49634 0a11b3f8-b1ba-4649-815b-8117442236d 5 10c04472-7cc0-4247-aa7b-2f166e8d4ab1 13a0737b-9859-4654-9a45-df034a2d1098 1532a693-b1e7-438e-b125-9e79953baf73 1b860b67-d31d-4633-98ba-5b62d2db1f2e 240b1487-b592-4c5d-a30c-1910c17512a0 2a25228b-7860-4342-bb88- b75d0068cfa1 3417cdde-16a7-4db0-b717-319ad7459a32 41ebdb00-9cce-47bf-9849-e1a203f915b1 45a7e1bb-62d0-4ec9-871b-53320f031d69 45e40b3c-4cde-4c69-8980-6b067e1c6f45 4bebffa0-a635-4880-81e7-6da162f65ae0 4ce91155-325f -4a85-95cc-3b8504a24a19 4eb3e323-8278-48be-a2d7-3288285cfe0e 4f576145-ef6e-4408-8781-a229e8c73506 587912ee-e688-4ef0-95cc-38c1e44a0108 5d5bfbf6-da93-4044-9263-c716aa699b0d 66625dce-c24c-4479-a576-8eef0fee4055 67 28cfbd-54ad-493d-9de5-f29189c6509b 706ade56-cdc8-4a20-8836-637837ad4d84 71927cfe-ac03-440c-ac9e-227aa6c330c6 71a14d97-0ee3-42b1-a47a-63589504c0a1 7b7777e0-a9d8-4417-a8ea-2aa66644d02b 81b048c3-4a70-4eaa-b17f-aa9a f440d9fb 8432d426-a68f-4757-9aef-7501f70c46f7 8b4e2ab3-82e8-4baa-b7c0-abaf6a83d0fa 8f81fc2c-e2c6-4d64-ae5f-c86d25f5687f 93542f94-9dec-4cbf-9200-3ce926d3e560 97b89ca2-2335-46e6-88eb-ed74d084789f a46434bf-26b5-416 8-bb6c-69cc328e98ad a4648426-4d7f-4bf7-b40e-4df1fbc6cf10 a68ed82d-f2de-4cd9-aa1b-264e78a5ed26 a8f2055d-406f-47f7-b8c6-4abc7de7ffac aaa40b67-8c2b-4c29-bdf7-8d0802b89c49 ae11be0f-8053-419e-ad2e-0ba960870dc6 af7c56 6e-ed23-4e2a-8696-668a598e7a7b b5e67f77-2a3e-4366-aa0b-6705eb254282 bcd4c9eb-13c9-436b-a355-37f237969de5 c1d9ce94-7b19-49ef-8159-39450968e50e c7192d48-3db5-4a49-a849-ec8a814728e9 cefa01d7-5120-4b70-9cf0-dedb52b2 9753 d02c33af-6d30-40e4-a678-85b3db567ae1 d5b2f49c-1cdb-4086-83f5-9a6993d8b0ef d6ddb601-30ba-4b3f-aa63-62afff9b361c dbfddd53-5a09-41eb-bbf2-33916b240fc1 e8746562-a990-40a3-b142-466e0d77c6f4 ec44d001-484b-4fe2-b7 e3-94c060c22b0f ee2734c5-21a0-42ec-813c-be174c7b751f f21577cc-78b0-47dd-b342-0de071c77743 f2d00d0a-0f4b-4863-81c0-8172084bcf38 f2fb2f8b-33e3-40f5-8859-d2f0a99b376e f327ce83-d94f-4b56-9e34-d0fa3ae5de9f f4b120db-e 21f-4d74-87cc-abd2a88779c3] I0708 15:46:35.468207 1 services_controller.go:223] Finished syncing service deployment-2pod-123-1 on namespace cluster-density-503fcc1a-1066-44d9-bef7-43292b046b06-123 : 15.673423702s E0708 15:46:35.468297 1 runtime.go:78] Observed a panic: "invalid memory address or nil pointer dereference" (runtime error: invalid memory address or nil pointer dereference) goroutine 8824 [running]: k8s.io/apimachinery/pkg/util/runtime.logPanic(0x18f2520, 0x28a4910) /go/src/github.com/openshift/ovn-kubernetes/go-controller/vendor/k8s.io/apimachinery/pkg/util/runtime/runtime.go:74 +0x95 k8s.io/apimachinery/pkg/util/runtime.HandleCrash(0x0, 0x0, 0x0) /go/src/github.com/openshift/ovn-kubernetes/go-controller/vendor/k8s.io/apimachinery/pkg/util/runtime/runtime.go:48 +0x86 panic(0x18f2520, 0x28a4910) /usr/lib/golang/src/runtime/panic.go:965 +0x1b9 k8s.io/api/core/v1.(*Service).GetObjectKind(0x0, 0x7fa626221c00, 0x0) :1 +0x5 k8s.io/client-go/tools/reference.GetReference(0xc0002bb490, 0x1d3f9a0, 0x0, 0x7fa5dc16fb38, 0x0, 0x0) /go/src/github.com/openshift/ovn-kubernetes/go-controller/vendor/k8s.io/client-go/tools/reference/ref.go:59 +0x14d k8s.io/client-go/tools/record.(*recorderImpl).generateEvent(0xc000f673c0, 0x1d3f9a0, 0x0, 0x0, 0xc031e556dbe7c809, 0x65dd7aa6698, 0x28e28c0, 0x1afbbca, 0x7, 0x1b1bd64, ...) /go/src/github.com/openshift/ovn-kubernetes/go-controller/vendor/k8s.io/client-go/tools/record/event.go:327 +0x5d k8s.io/client-go/tools/record.(*recorderImpl).Event(0xc000f673c0, 0x1d3f9a0, 0x0, 0x1afbbca, 0x7, 0x1b1bd64, 0x1d, 0xc0105b8000, 0x1ec3) /go/src/github.com/openshift/ovn-kubernetes/go-controller/vendor/k8s.io/client-go/tools/record/event.go:349 +0xc5 k8s.io/client-go/tools/record.(*recorderImpl).Eventf(0xc000f673c0, 0x1d3f9a0, 0x0, 0x1afbbca, 0x7, 0x1b1bd64, 0x1d, 0x1b62102, 0x41, 0xc00df6c720, ...) /go/src/github.com/openshift/ovn-kubernetes/go-controller/vendor/k8s.io/client-go/tools/record/event.go:353 +0xca github.com/ovn-org/ovn-kubernetes/go-controller/pkg/ovn/controller/services.(*Controller).syncServices(0xc00176b4d0, 0xc00a217d60, 0x4e, 0x0, 0x0) /go/src/github.com/openshift/ovn-kubernetes/go-controller/pkg/ovn/controller/services/services_controller.go:246 +0x682 github.com/ovn-org/ovn-kubernetes/go-controller/pkg/ovn/controller/services.(*Controller).processNextWorkItem(0xc00176b4d0, 0x203000) /go/src/github.com/openshift/ovn-kubernetes/go-controller/pkg/ovn/controller/services/services_controller.go:184 +0xcd github.com/ovn-org/ovn-kubernetes/go-controller/pkg/ovn/controller/services.(*Controller).worker(...) /go/src/github.com/openshift/ovn-kubernetes/go-controller/pkg/ovn/controller/services/services_controller.go:173 k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0xc002f319b0) /go/src/github.com/openshift/ovn-kubernetes/go-controller/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:155 +0x5f k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc002f319b0, 0x1d339c0, 0xc0024048a0, 0xc001987701, 0xc000210780) /go/src/github.com/openshift/ovn-kubernetes/go-controller/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:156 +0x9b k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc002f319b0, 0x3b9aca00, 0x0, 0xc001987701, 0xc000210780) /go/src/github.com/openshift/ovn-kubernetes/go-controller/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:133 +0x98 k8s.io/apimachinery/pkg/util/wait.Until(0xc002f319b0, 0x3b9aca00, 0xc000210780) /go/src/github.com/openshift/ovn-kubernetes/go-controller/vendor/k8s.io/apimachinery/pkg/util/wait/wait.go:90 +0x4d created by github.com/ovn-org/ovn-kubernetes/go-controller/pkg/ovn/controller/services.(*Controller).Run /go/src/github.com/openshift/ovn-kubernetes/go-controller/pkg/ovn/controller/services/services_controller.go:161 +0x3b1 panic: runtime error: invalid memory address or nil pointer dereference [recovered] panic: runtime error: invalid memory address or nil pointer dereference [signal SIGSEGV: segmentation violation code=0x1 addr=0x0 pc=0xd3aa05] Signed-off-by: Dan Williams (cherry picked from commit 9d950cce29fffe6a6e074985f03b735c080cf323) --- .../pkg/ovn/controller/services/services_controller.go | 8 ++++++-- go-controller/pkg/ovn/controller/services/utils.go | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/go-controller/pkg/ovn/controller/services/services_controller.go b/go-controller/pkg/ovn/controller/services/services_controller.go index f5d7418c77..11770b811f 100644 --- a/go-controller/pkg/ovn/controller/services/services_controller.go +++ b/go-controller/pkg/ovn/controller/services/services_controller.go @@ -243,8 +243,12 @@ func (c *Controller) syncServices(key string) error { if err != nil || !util.ServiceTypeHasClusterIP(service) || !util.IsClusterIPSet(service) { err = deleteVIPsFromAllOVNBalancers(vipsTracked, name, namespace) if err != nil { - c.eventRecorder.Eventf(service, v1.EventTypeWarning, "FailedToDeleteOVNLoadBalancer", - "Error trying to delete the OVN LoadBalancer for Service %s/%s: %v", name, namespace, err) + // If the service wasn't found, don't panic sending an + // an event after cleaning it up + if service != nil { + c.eventRecorder.Eventf(service, v1.EventTypeWarning, "FailedToDeleteOVNLoadBalancer", + "Error trying to delete the OVN LoadBalancer for Service %s/%s: %v", name, namespace, err) + } return err } // Delete the Service form the Service Tracker diff --git a/go-controller/pkg/ovn/controller/services/utils.go b/go-controller/pkg/ovn/controller/services/utils.go index 1f05c05bca..add6f8de5d 100644 --- a/go-controller/pkg/ovn/controller/services/utils.go +++ b/go-controller/pkg/ovn/controller/services/utils.go @@ -90,7 +90,7 @@ func deleteVIPsFromNonIdlingOVNBalancers(vips sets.String, name, namespace strin for proto := range foundProtocols { if err := loadbalancer.DeleteLoadBalancerVIPs(lbsPerProtocol[proto].List(), vipsPerProtocol[proto].List()); err != nil { - klog.Errorf("Error deleting VIP %v on OVN LoadBalancer %v", vipsPerProtocol[proto].List(), lbsPerProtocol[proto].List()) + klog.Errorf("Error deleting VIP %v on OVN LoadBalancer %v: %v", vipsPerProtocol[proto].List(), lbsPerProtocol[proto].List(), err) return err } }