From 958aab96fb1e3daa4e2f60412ab32ea673690949 Mon Sep 17 00:00:00 2001 From: bobz965 Date: Mon, 25 Dec 2023 12:09:51 +0800 Subject: [PATCH] fix: add err log (#3572) * fix: add err log * fix: gw chassis creation --------- Signed-off-by: bobz965 --- cmd/daemon/cniserver.go | 2 +- pkg/controller/external_gw.go | 43 +++++++++++++++++-------------- pkg/controller/node.go | 17 ++++++++++-- pkg/controller/ovn_ic.go | 30 +++++++++++---------- pkg/daemon/ovs_linux.go | 13 +++++++--- pkg/ovs/ovn-nb-gateway_chassis.go | 2 +- pkg/util/const.go | 13 ++++++---- 7 files changed, 74 insertions(+), 46 deletions(-) diff --git a/cmd/daemon/cniserver.go b/cmd/daemon/cniserver.go index 58d6a105df0..0dba538e825 100644 --- a/cmd/daemon/cniserver.go +++ b/cmd/daemon/cniserver.go @@ -49,7 +49,7 @@ func CmdMain() { util.LogFatalAndExit(err, "failed to initialize config") } - if err := Retry(util.ChasRetryTime, util.ChasRetryIntev, initChassisAnno, config); err != nil { + if err := Retry(util.ChassisRetryMaxTimes, util.ChassisCniDaemonRetryInterval, initChassisAnno, config); err != nil { util.LogFatalAndExit(err, "failed to initialize ovn chassis annotation") } diff --git a/pkg/controller/external_gw.go b/pkg/controller/external_gw.go index 0350cb84529..180e8c36339 100644 --- a/pkg/controller/external_gw.go +++ b/pkg/controller/external_gw.go @@ -92,13 +92,14 @@ func (c *Controller) removeExternalGateway() error { if len(no.Labels) == 0 { op = "add" } - no.Labels[util.ExGatewayLabel] = "false" - raw, _ := json.Marshal(no.Labels) - patchPayload := fmt.Sprintf(patchPayloadTemplate, op, raw) - _, err = c.config.KubeClient.CoreV1().Nodes().Patch(context.Background(), no.Name, types.JSONPatchType, []byte(patchPayload), metav1.PatchOptions{}, "") - if err != nil { - klog.Errorf("patch external gw node %s failed %v", no.Name, err) - return err + if no.Labels[util.ExGatewayLabel] != "false" { + no.Labels[util.ExGatewayLabel] = "false" + raw, _ := json.Marshal(no.Labels) + patchPayload := fmt.Sprintf(patchPayloadTemplate, op, raw) + if _, err = c.config.KubeClient.CoreV1().Nodes().Patch(context.Background(), no.Name, types.JSONPatchType, []byte(patchPayload), metav1.PatchOptions{}, ""); err != nil { + klog.Errorf("patch external gw node %s failed %v", no.Name, err) + return err + } } } @@ -216,7 +217,7 @@ func (c *Controller) getGatewayChassis(config map[string]string) ([]string, erro nodes, err := c.nodesLister.List(labels.Everything()) if err != nil { klog.Errorf("failed to list nodes, %v", err) - return chassises, err + return nil, err } gwNodes := make([]string, 0, len(nodes)) for _, node := range nodes { @@ -238,7 +239,7 @@ func (c *Controller) getGatewayChassis(config map[string]string) ([]string, erro cachedNode, err := c.nodesLister.Get(gw) if err != nil { klog.Errorf("failed to get gw node %s, %v", gw, err) - return chassises, err + return nil, err } node := cachedNode.DeepCopy() patchPayloadTemplate := `[{ @@ -250,31 +251,33 @@ func (c *Controller) getGatewayChassis(config map[string]string) ([]string, erro if len(node.Labels) == 0 { op = "add" } - node.Labels[util.ExGatewayLabel] = "true" - raw, _ := json.Marshal(node.Labels) - patchPayload := fmt.Sprintf(patchPayloadTemplate, op, raw) - _, err = c.config.KubeClient.CoreV1().Nodes().Patch(context.Background(), gw, types.JSONPatchType, []byte(patchPayload), metav1.PatchOptions{}, "") - if err != nil { - klog.Errorf("patch external gw node %s failed %v", gw, err) - return chassises, err + if node.Labels[util.ExGatewayLabel] != "true" { + node.Labels[util.ExGatewayLabel] = "true" + raw, _ := json.Marshal(node.Labels) + patchPayload := fmt.Sprintf(patchPayloadTemplate, op, raw) + if _, err = c.config.KubeClient.CoreV1().Nodes().Patch(context.Background(), gw, types.JSONPatchType, []byte(patchPayload), metav1.PatchOptions{}, ""); err != nil { + klog.Errorf("patch external gw node %s failed %v", gw, err) + return nil, err + } } annoChassisName := node.Annotations[util.ChassisAnnotation] if annoChassisName == "" { err := fmt.Errorf("node %s has no chassis annotation, kube-ovn-cni not ready", gw) klog.Error(err) - return chassises, err + return nil, err } klog.Infof("get node %s chassis: %s", gw, annoChassisName) chassis, err := c.OVNSbClient.GetChassis(annoChassisName, false) if err != nil { klog.Errorf("failed to get node %s chassis: %s, %v", node.Name, annoChassisName, err) - return chassises, err + return nil, err } chassises = append(chassises, chassis.Name) } if len(chassises) == 0 { - klog.Error("no available external gw") - return chassises, fmt.Errorf("no available external gw") + err := fmt.Errorf("no available external gw") + klog.Error(err) + return nil, err } return chassises, nil } diff --git a/pkg/controller/node.go b/pkg/controller/node.go index c8880fc1985..f0063d08324 100644 --- a/pkg/controller/node.go +++ b/pkg/controller/node.go @@ -195,6 +195,7 @@ func (c *Controller) handleAddNode(key string) error { if k8serrors.IsNotFound(err) { return nil } + klog.Errorf("failed to get node %s: %v", key, err) return err } node := cachedNode.DeepCopy() @@ -252,6 +253,7 @@ func (c *Controller) handleAddNode(key string) error { ipStr := util.GetStringIP(v4IP, v6IP) if err := c.OVNNbClient.CreateBareLogicalSwitchPort(c.config.NodeSwitch, portName, ipStr, mac); err != nil { + klog.Errorf("failed to create logical switch port %s: %v", portName, err) return err } @@ -290,11 +292,13 @@ func (c *Controller) handleAddNode(key string) error { } if err = c.deletePolicyRouteForLocalDNSCacheOnNode(node.Name, af); err != nil { + klog.Errorf("failed to delete policy route for node %s: %v", node.Name, err) return err } if c.config.NodeLocalDNSIP != "" { if err = c.addPolicyRouteForLocalDNSCacheOnNode(portName, ip, node.Name, af); err != nil { + klog.Errorf("failed to add policy route for node %s: %v", node.Name, err) return err } } @@ -366,10 +370,15 @@ func (c *Controller) handleAddNode(key string) error { } if err := c.UpdateChassisTag(node); err != nil { + klog.Errorf("failed to update chassis tag for node %s: %v", node.Name, err) return err } - return c.retryDelDupChassis(util.ChasRetryTime, util.ChasRetryIntev+2, c.cleanDuplicatedChassis, node) + if err := c.retryDelDupChassis(util.ChassisRetryMaxTimes, util.ChassisControllerRetryInterval, c.cleanDuplicatedChassis, node); err != nil { + klog.Errorf("failed to clean duplicated chassis for node %s: %v", node.Name, err) + return err + } + return nil } func (c *Controller) handleNodeAnnotationsForProviderNetworks(node *v1.Node) error { @@ -598,6 +607,7 @@ func (c *Controller) handleUpdateNode(key string) error { if k8serrors.IsNotFound(err) { return nil } + klog.Errorf("failed to get node %s: %v", key, err) return err } @@ -613,9 +623,11 @@ func (c *Controller) handleUpdateNode(key string) error { } if err := c.UpdateChassisTag(node); err != nil { + klog.Errorf("failed to update chassis tag for node %s: %v", node.Name, err) return err } - if err := c.retryDelDupChassis(util.ChasRetryTime, util.ChasRetryIntev+2, c.cleanDuplicatedChassis, node); err != nil { + if err := c.retryDelDupChassis(util.ChassisRetryMaxTimes, util.ChassisControllerRetryInterval, c.cleanDuplicatedChassis, node); err != nil { + klog.Errorf("failed to clean duplicated chassis for node %s: %v", node.Name, err) return err } @@ -880,6 +892,7 @@ func (c *Controller) retryDelDupChassis(attempts, sleep int, f func(node *v1.Nod if err == nil { return } + klog.Errorf("failed to delete duplicated chassis for node %s: %v", node.Name, err) if i >= (attempts - 1) { break } diff --git a/pkg/controller/ovn_ic.go b/pkg/controller/ovn_ic.go index 64c54b06531..a381b04e77f 100644 --- a/pkg/controller/ovn_ic.go +++ b/pkg/controller/ovn_ic.go @@ -155,13 +155,14 @@ func (c *Controller) removeInterConnection(azName string) error { if len(no.Labels) == 0 { op = "add" } - no.Labels[util.ICGatewayLabel] = "false" - raw, _ := json.Marshal(no.Labels) - patchPayload := fmt.Sprintf(patchPayloadTemplate, op, raw) - _, err = c.config.KubeClient.CoreV1().Nodes().Patch(context.Background(), no.Name, types.JSONPatchType, []byte(patchPayload), metav1.PatchOptions{}, "") - if err != nil { - klog.Errorf("patch ic gw node %s failed %v", no.Name, err) - return err + if no.Labels[util.ICGatewayLabel] != "false" { + no.Labels[util.ICGatewayLabel] = "false" + raw, _ := json.Marshal(no.Labels) + patchPayload := fmt.Sprintf(patchPayloadTemplate, op, raw) + if _, err = c.config.KubeClient.CoreV1().Nodes().Patch(context.Background(), no.Name, types.JSONPatchType, []byte(patchPayload), metav1.PatchOptions{}, ""); err != nil { + klog.Errorf("patch ic gw node %s failed %v", no.Name, err) + return err + } } } @@ -242,13 +243,14 @@ func (c *Controller) establishInterConnection(config map[string]string) error { if len(node.Labels) == 0 { op = "add" } - node.Labels[util.ICGatewayLabel] = "true" - raw, _ := json.Marshal(node.Labels) - patchPayload := fmt.Sprintf(patchPayloadTemplate, op, raw) - _, err = c.config.KubeClient.CoreV1().Nodes().Patch(context.Background(), gw, types.JSONPatchType, []byte(patchPayload), metav1.PatchOptions{}, "") - if err != nil { - klog.Errorf("patch gw node %s failed %v", gw, err) - return err + if node.Labels[util.ICGatewayLabel] != "true" { + node.Labels[util.ICGatewayLabel] = "true" + raw, _ := json.Marshal(node.Labels) + patchPayload := fmt.Sprintf(patchPayloadTemplate, op, raw) + if _, err = c.config.KubeClient.CoreV1().Nodes().Patch(context.Background(), gw, types.JSONPatchType, []byte(patchPayload), metav1.PatchOptions{}, ""); err != nil { + klog.Errorf("patch gw node %s failed %v", gw, err) + return err + } } } diff --git a/pkg/daemon/ovs_linux.go b/pkg/daemon/ovs_linux.go index 0c8e105d9a2..9fb650be2f9 100644 --- a/pkg/daemon/ovs_linux.go +++ b/pkg/daemon/ovs_linux.go @@ -620,11 +620,18 @@ func (c *Controller) loopOvnExt0Check() { cachedEip, err := c.ovnEipsLister.Get(portName) if err != nil { if k8serrors.IsNotFound(err) { - if val, ok := node.Labels[util.NodeExtGwLabel]; !ok || (val == "false") { - // not gw node or already clean + val, ok := node.Labels[util.NodeExtGwLabel] + if !ok { + // not gw node before return } - needClean = true + if val == "false" { + // already clean + return + } + if val == "true" { + needClean = true + } } else { klog.Errorf("failed to get ecmp gateway ovn eip, %v", err) return diff --git a/pkg/ovs/ovn-nb-gateway_chassis.go b/pkg/ovs/ovn-nb-gateway_chassis.go index 87c833d1e32..af9dea97ca6 100644 --- a/pkg/ovs/ovn-nb-gateway_chassis.go +++ b/pkg/ovs/ovn-nb-gateway_chassis.go @@ -118,7 +118,7 @@ func (c *OVNNbClient) CreateGatewayChassisesOp(lrpName string, chassises []strin for i, chassisName := range chassises { gwChassisName := lrpName + "-" + chassisName - gwChassis, err := c.GetGatewayChassis(chassisName, true) + gwChassis, err := c.GetGatewayChassis(gwChassisName, true) if err != nil { klog.Error(err) return nil, err diff --git a/pkg/util/const.go b/pkg/util/const.go index 0a1fef60c40..81e1d27575e 100644 --- a/pkg/util/const.go +++ b/pkg/util/const.go @@ -202,11 +202,14 @@ const ( InternalType = "internal-port" DpdkType = "dpdk-port" - HostnameEnv = "KUBE_NODE_NAME" - ChasRetryTime = 5 - ChasRetryIntev = 1 - VM = "VirtualMachine" - VMInstance = "VirtualMachineInstance" + HostnameEnv = "KUBE_NODE_NAME" + + ChassisRetryMaxTimes = 5 + ChassisCniDaemonRetryInterval = 1 + ChassisControllerRetryInterval = 3 + + VM = "VirtualMachine" + VMInstance = "VirtualMachineInstance" MirrorControlAnnotation = "ovn.kubernetes.io/mirror" MirrorDefaultName = "m0"