From a7a11f0301adc92a4f4d0513bd393c8a5ccded22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E7=9D=BF=E5=8D=8E?= Date: Wed, 1 Jun 2022 17:54:11 +0800 Subject: [PATCH] fix: Do not Recreate Logical_Router_Port when Vpc recreated (#1570) * fix issue 1569 * rename some local variable --- pkg/controller/vpc.go | 42 ++++++++++++++ pkg/ovs/ovn-nb-logical_router.go | 31 ++++++++++ pkg/ovs/ovn-nb-logical_router_port.go | 82 +++++++++++++++++++++++++++ pkg/ovs/ovn.go | 19 +++++++ pkg/ovs/util.go | 8 +++ pkg/ovsdb/client/client.go | 1 + 6 files changed, 183 insertions(+) create mode 100644 pkg/ovs/ovn-nb-logical_router.go create mode 100644 pkg/ovs/ovn-nb-logical_router_port.go diff --git a/pkg/controller/vpc.go b/pkg/controller/vpc.go index b3a65c15cda..3f5ea18f23b 100644 --- a/pkg/controller/vpc.go +++ b/pkg/controller/vpc.go @@ -162,6 +162,43 @@ func (c *Controller) handleUpdateVpcStatus(key string) error { return nil } +func (c *Controller) reconcileRouterPorts(vpc *kubeovnv1.Vpc) error { + subnets, _, err := c.getVpcSubnets(vpc) + if err != nil { + klog.ErrorS(err, "unable to get related subnets", "vpc", vpc.Name) + return err + } + + router := vpc.Name + for _, subnetName := range subnets { + routerPortName := ovs.LogicalRouterPortName(router, subnetName) + exists, err := c.ovnClient.LogicalRouterPortExists(routerPortName) + if err != nil { + return err + } + + if !exists { + subnet, err := c.subnetsLister.Get(subnetName) + if err != nil { + if k8serrors.IsNotFound(err) { + continue + } + klog.ErrorS(err, "unable to get subnet", "subnet", subnetName) + return err + } + + klog.V(1).InfoS("router port not exists, trying to create", "vpc", vpc.Name, "subnet", subnetName) + + networks := util.GetIpAddrWithMask(subnet.Spec.Gateway, subnet.Spec.CIDRBlock) + if err := c.ovnClient.AddLogicalRouterPort(router, routerPortName, "", networks); err != nil { + klog.ErrorS(err, "unable to create router port", "vpc", vpc.Name, "subnet", subnetName) + return err + } + } + } + return nil +} + type VpcLoadBalancer struct { TcpLoadBalancer string TcpSessLoadBalancer string @@ -271,6 +308,11 @@ func (c *Controller) handleAddOrUpdateVpc(key string) error { return err } + if err := c.reconcileRouterPorts(vpc); err != nil { + klog.ErrorS(err, "unable to reconcileRouterPorts") + return err + } + var newPeers []string for _, peering := range vpc.Spec.VpcPeerings { if err = util.CheckCidrs(peering.LocalConnectIP); err != nil { diff --git a/pkg/ovs/ovn-nb-logical_router.go b/pkg/ovs/ovn-nb-logical_router.go new file mode 100644 index 00000000000..656b74dc727 --- /dev/null +++ b/pkg/ovs/ovn-nb-logical_router.go @@ -0,0 +1,31 @@ +package ovs + +import ( + "context" + "fmt" + + "github.com/ovn-org/libovsdb/client" + + "github.com/kubeovn/kube-ovn/pkg/ovsdb/ovnnb" +) + +func (c OvnClient) GetLogicalRouter(name string, ignoreNotFound bool) (*ovnnb.LogicalRouter, error) { + predicate := func(model *ovnnb.LogicalRouter) bool { + return model.Name == name + } + // Logical_Router has no indexes defined in the schema + var result []*ovnnb.LogicalRouter + if err := c.ovnNbClient.WhereCache(predicate).List(context.TODO(), &result); err != nil || len(result) == 0 { + if ignoreNotFound && (err == client.ErrNotFound || len(result) == 0) { + return nil, nil + } + return nil, fmt.Errorf("failed to get logical router %s: %v", name, err) + } + + return result[0], nil +} + +func (c OvnClient) LogicalRouterExists(name string) (bool, error) { + lr, err := c.GetLogicalRouter(name, true) + return lr != nil, err +} diff --git a/pkg/ovs/ovn-nb-logical_router_port.go b/pkg/ovs/ovn-nb-logical_router_port.go new file mode 100644 index 00000000000..507a90d4014 --- /dev/null +++ b/pkg/ovs/ovn-nb-logical_router_port.go @@ -0,0 +1,82 @@ +package ovs + +import ( + "context" + "fmt" + "strings" + + "github.com/ovn-org/libovsdb/client" + "github.com/ovn-org/libovsdb/model" + "github.com/ovn-org/libovsdb/ovsdb" + + ovsclient "github.com/kubeovn/kube-ovn/pkg/ovsdb/client" + "github.com/kubeovn/kube-ovn/pkg/ovsdb/ovnnb" + "github.com/kubeovn/kube-ovn/pkg/util" +) + +func (c OvnClient) GetLogicalRouterPort(name string, ignoreNotFound bool) (*ovnnb.LogicalRouterPort, error) { + lrp := &ovnnb.LogicalRouterPort{Name: name} + if err := c.ovnNbClient.Get(context.TODO(), lrp); err != nil { + if ignoreNotFound && err == client.ErrNotFound { + return nil, nil + } + return nil, fmt.Errorf("failed to get logical router port %s: %v", name, err) + } + + return lrp, nil +} + +func (c OvnClient) AddLogicalRouterPort(lr, name, mac, networks string) error { + router, err := c.GetLogicalRouter(lr, false) + if err != nil { + return err + } + + if mac == "" { + mac = util.GenerateMac() + } + + var ops []ovsdb.Operation + + lrp := &ovnnb.LogicalRouterPort{ + UUID: ovsclient.NamedUUID(), + Name: name, + MAC: mac, + Networks: strings.Split(networks, ","), + ExternalIDs: map[string]string{"vendor": util.CniTypeName}, + } + + // ensure there is no port in the same name, before we create it in the transaction + waitOp := ConstructWaitForNameNotExistsOperation(name, "Logical_Router_Port") + ops = append(ops, waitOp) + + createOps, err := c.ovnNbClient.Create(lrp) + if err != nil { + return err + } + ops = append(ops, createOps...) + + mutationOps, err := c.ovnNbClient. + Where(router). + Mutate(router, + model.Mutation{ + Field: &router.Ports, + Mutator: ovsdb.MutateOperationInsert, + Value: []string{lrp.UUID}, + }, + ) + if err != nil { + return err + } + ops = append(ops, mutationOps...) + + if err := Transact(c.ovnNbClient, "lrp-add", ops, c.ovnNbClient.Timeout); err != nil { + return fmt.Errorf("failed to create logical router port %s: %v", name, err) + } + return nil +} + +func (c OvnClient) LogicalRouterPortExists(name string) (bool, error) { + lrp, err := c.GetLogicalRouterPort(name, true) + return lrp != nil, err +} diff --git a/pkg/ovs/ovn.go b/pkg/ovs/ovn.go index 7cedf7231b2..4a75f478cbc 100644 --- a/pkg/ovs/ovn.go +++ b/pkg/ovs/ovn.go @@ -52,6 +52,8 @@ const ( Policy = "--policy" PolicyDstIP = "dst-ip" PolicySrcIP = "src-ip" + + OVSDBWaitTimeout = 0 ) // NewLegacyClient init a legacy ovn client @@ -118,3 +120,20 @@ func Transact(c client.Client, method string, operations []ovsdb.Operation, time return nil } + +func ConstructWaitForNameNotExistsOperation(name string, table string) ovsdb.Operation { + return ConstructWaitForUniqueOperation(table, "name", name) +} + +func ConstructWaitForUniqueOperation(table string, column string, value interface{}) ovsdb.Operation { + timeout := OVSDBWaitTimeout + return ovsdb.Operation{ + Op: ovsdb.OperationWait, + Table: table, + Timeout: &timeout, + Where: []ovsdb.Condition{{Column: column, Function: ovsdb.ConditionEqual, Value: value}}, + Columns: []string{column}, + Until: "!=", + Rows: []ovsdb.Row{{column: value}}, + } +} diff --git a/pkg/ovs/util.go b/pkg/ovs/util.go index f825cb02ba4..ddcee0ad41f 100644 --- a/pkg/ovs/util.go +++ b/pkg/ovs/util.go @@ -23,3 +23,11 @@ func trimCommandOutput(raw []byte) string { output := strings.TrimSpace(string(raw)) return strings.Trim(output, "\"") } + +func LogicalRouterPortName(lr, ls string) string { + return fmt.Sprintf("%s-%s", lr, ls) +} + +func LogicalSwitchPortName(lr, ls string) string { + return fmt.Sprintf("%s-%s", ls, lr) +} diff --git a/pkg/ovsdb/client/client.go b/pkg/ovsdb/client/client.go index e5625140e1d..22420e63231 100644 --- a/pkg/ovsdb/client/client.go +++ b/pkg/ovsdb/client/client.go @@ -84,6 +84,7 @@ func NewNbClient(addr string, timeout int) (client.Client, error) { monitorOpts := []client.MonitorOption{ client.WithTable(&ovnnb.LogicalRouter{}), + client.WithTable(&ovnnb.LogicalRouterPort{}), client.WithTable(&ovnnb.LogicalRouterPolicy{}), client.WithTable(&ovnnb.LogicalSwitchPort{}), client.WithTable(&ovnnb.PortGroup{}),