diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index 139d30890f7..fa81063c34d 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -2,6 +2,7 @@ package controller import ( "context" + "fmt" "sync" "time" @@ -868,17 +869,14 @@ func (c *Controller) startWorkers(ctx context.Context) { go wait.Until(c.runAddSubnetWorker, time.Second, ctx.Done()) go wait.Until(c.runAddVlanWorker, time.Second, ctx.Done()) go wait.Until(c.runAddNamespaceWorker, time.Second, ctx.Done()) - for { - klog.Infof("wait for %s and %s ready", c.config.DefaultLogicalSwitch, c.config.NodeSwitch) - time.Sleep(3 * time.Second) - lss, err := c.ovnLegacyClient.ListLogicalSwitch(c.config.EnableExternalVpc) - if err != nil { - util.LogFatalAndExit(err, "failed to list logical switch") - } - - if util.IsStringIn(c.config.DefaultLogicalSwitch, lss) && util.IsStringIn(c.config.NodeSwitch, lss) && c.addNamespaceQueue.Len() == 0 { - break - } + err := wait.PollUntil(3*time.Second, func() (done bool, err error) { + subnets := []string{c.config.DefaultLogicalSwitch, c.config.NodeSwitch} + klog.Infof("wait for subnets %v ready", subnets) + + return c.allSubnetReady(subnets...) + }, ctx.Done()) + if err != nil { + klog.Fatalf("wait default/join subnet ready error: %v", err) } go wait.Until(c.runAddSgWorker, time.Second, ctx.Done()) @@ -1009,8 +1007,6 @@ func (c *Controller) startWorkers(ctx context.Context) { go wait.Until(c.CheckNodePortGroup, time.Duration(c.config.NodePgProbeTime)*time.Minute, ctx.Done()) } - go wait.Until(c.syncVmLiveMigrationPort, 15*time.Second, ctx.Done()) - go wait.Until(c.runAddVirtualIpWorker, time.Second, ctx.Done()) go wait.Until(c.runUpdateVirtualIpWorker, time.Second, ctx.Done()) go wait.Until(c.runDelVirtualIpWorker, time.Second, ctx.Done()) @@ -1040,3 +1036,18 @@ func (c *Controller) startWorkers(ctx context.Context) { go wait.Until(c.runDelPodAnnotatedIptablesFipWorker, time.Second, ctx.Done()) } } + +func (c *Controller) allSubnetReady(subnets ...string) (bool, error) { + for _, lsName := range subnets { + exist, err := c.ovnClient.LogicalSwitchExists(lsName) + if err != nil { + return false, fmt.Errorf("check logical switch %s exist: %v", lsName, err) + } + + if !exist { + return false, nil + } + } + + return true, nil +} diff --git a/pkg/controller/controller_test.go b/pkg/controller/controller_test.go new file mode 100644 index 00000000000..bef201d7edf --- /dev/null +++ b/pkg/controller/controller_test.go @@ -0,0 +1,85 @@ +package controller + +import ( + "testing" + + "github.com/golang/mock/gomock" + "github.com/stretchr/testify/require" + "k8s.io/client-go/util/workqueue" + + mockovs "github.com/kubeovn/kube-ovn/mocks/pkg/ovs" + "github.com/kubeovn/kube-ovn/pkg/client/clientset/versioned/fake" + informerfactory "github.com/kubeovn/kube-ovn/pkg/client/informers/externalversions" + kubeovninformer "github.com/kubeovn/kube-ovn/pkg/client/informers/externalversions/kubeovn/v1" +) + +type fakeControllerInformers struct { + vpcInformer kubeovninformer.VpcInformer + sbunetInformer kubeovninformer.SubnetInformer +} + +type fakeController struct { + fakeController *Controller + fakeinformers *fakeControllerInformers + mockOvnClient *mockovs.MockOvnClient +} + +func alwaysReady() bool { return true } + +func newFakeController(t *testing.T) *fakeController { + /* kube ovn fake client */ + kubeovnClient := fake.NewSimpleClientset() + kubeovnInformerFactory := informerfactory.NewSharedInformerFactory(kubeovnClient, 0) + vpcInformer := kubeovnInformerFactory.Kubeovn().V1().Vpcs() + sbunetInformer := kubeovnInformerFactory.Kubeovn().V1().Subnets() + + fakeInformers := &fakeControllerInformers{ + vpcInformer: vpcInformer, + sbunetInformer: sbunetInformer, + } + + /* ovn fake client */ + mockOvnClient := mockovs.NewMockOvnClient(gomock.NewController(t)) + + ctrl := &Controller{ + vpcsLister: vpcInformer.Lister(), + vpcSynced: alwaysReady, + subnetsLister: sbunetInformer.Lister(), + subnetSynced: alwaysReady, + ovnClient: mockOvnClient, + syncVirtualPortsQueue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), ""), + } + + return &fakeController{ + fakeController: ctrl, + fakeinformers: fakeInformers, + mockOvnClient: mockOvnClient, + } +} + +func Test_allSubnetReady(t *testing.T) { + t.Parallel() + + fakeController := newFakeController(t) + ctrl := fakeController.fakeController + mockOvnClient := fakeController.mockOvnClient + + subnets := []string{"ovn-default", "join"} + + t.Run("all subnet ready", func(t *testing.T) { + mockOvnClient.EXPECT().LogicalSwitchExists(gomock.Any()).Return(true, nil).Times(2) + + ready, err := ctrl.allSubnetReady(subnets...) + require.NoError(t, err) + require.True(t, ready) + }) + + t.Run("some subnet are not ready", func(t *testing.T) { + mockOvnClient.EXPECT().LogicalSwitchExists(subnets[0]).Return(true, nil) + mockOvnClient.EXPECT().LogicalSwitchExists(subnets[1]).Return(false, nil) + + ready, err := ctrl.allSubnetReady(subnets...) + require.NoError(t, err) + require.False(t, ready) + }) +} diff --git a/pkg/controller/external-gw.go b/pkg/controller/external-gw.go index d61107d5458..37dc08727c0 100644 --- a/pkg/controller/external-gw.go +++ b/pkg/controller/external-gw.go @@ -119,15 +119,15 @@ func (c *Controller) removeExternalGateway() error { if !keepExternalSubnet { klog.Infof("delete external gateway switch %s", c.config.ExternalGatewaySwitch) - if err := c.ovnLegacyClient.DeleteGatewaySwitch(c.config.ExternalGatewaySwitch); err != nil { - klog.Errorf("failed to delete external gateway switch, %v", err) + if err := c.ovnClient.DeleteLogicalGatewaySwitch(util.ExternalGatewaySwitch, c.config.ClusterRouter); err != nil { + klog.Errorf("delete external gateway switch %s: %v", util.ExternalGatewaySwitch, err) return err } } else { klog.Infof("should keep provider network vlan underlay external gateway switch %s", c.config.ExternalGatewaySwitch) lrpName := fmt.Sprintf("%s-%s", c.config.ClusterRouter, c.config.ExternalGatewaySwitch) klog.Infof("delete logical router port %s", lrpName) - if err := c.ovnLegacyClient.DeleteLogicalRouterPort(lrpName); err != nil { + if err := c.ovnClient.DeleteLogicalRouterPort(lrpName); err != nil { klog.Errorf("failed to delete lrp %s, %v", lrpName, err) return err } @@ -160,10 +160,12 @@ func (c *Controller) establishExternalGateway(config map[string]string) error { klog.Infof("lrp %s exist", lrpName) return nil } - if err := c.ovnLegacyClient.CreateGatewaySwitch(c.config.ExternalGatewaySwitch, c.config.ExternalGatewayNet, c.config.ExternalGatewayVlanID, lrpIp, lrpMac, chassises); err != nil { - klog.Errorf("failed to create external gateway switch, %v", err) + + if err := c.ovnClient.CreateGatewayLogicalSwitch(c.config.ExternalGatewaySwitch, c.config.ClusterRouter, c.config.ExternalGatewayNet, lrpIp, lrpMac, c.config.ExternalGatewayVlanID, chassises...); err != nil { + klog.Errorf("create external gateway switch %s: %v", c.config.ExternalGatewaySwitch, err) return err } + return nil } diff --git a/pkg/controller/gc.go b/pkg/controller/gc.go index a1069c25ee3..62e0488a88b 100644 --- a/pkg/controller/gc.go +++ b/pkg/controller/gc.go @@ -15,6 +15,7 @@ import ( kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1" "github.com/kubeovn/kube-ovn/pkg/ovs" + "github.com/kubeovn/kube-ovn/pkg/ovsdb/ovnnb" "github.com/kubeovn/kube-ovn/pkg/util" ) @@ -52,26 +53,17 @@ func (c *Controller) gcLogicalRouterPort() error { return err } - var exceptPeerPorts []string + exceptPeerPorts := make(map[string]struct{}) for _, vpc := range vpcs { for _, peer := range vpc.Status.VpcPeerings { - exceptPeerPorts = append(exceptPeerPorts, fmt.Sprintf("%s-%s", vpc.Name, peer)) + exceptPeerPorts[fmt.Sprintf("%s-%s", vpc.Name, peer)] = struct{}{} } } - lrps, err := c.ovnLegacyClient.ListLogicalEntity("logical_router_port", "peer!=[]") - if err != nil { - klog.Errorf("failed to list logical router port, %v", err) + + if err = c.ovnClient.DeleteLogicalRouterPorts(nil, logicalRouterPortFilter(exceptPeerPorts)); err != nil { + klog.Errorf("delete non-existent peer logical router port: %v", err) return err } - for _, lrp := range lrps { - if !util.ContainsString(exceptPeerPorts, lrp) { - klog.Infof("gc logical router port %s", lrp) - if err = c.ovnLegacyClient.DeleteLogicalRouterPort(lrp); err != nil { - klog.Errorf("failed to delete logical router port %s, %v", lrp, err) - return err - } - } - } return nil } @@ -132,25 +124,27 @@ func (c *Controller) gcLogicalSwitch() error { subnetMap[s.Name] = s subnetNames = append(subnetNames, s.Name) } - lss, err := c.ovnLegacyClient.ListLogicalSwitch(c.config.EnableExternalVpc) + + lss, err := c.ovnClient.ListLogicalSwitch(c.config.EnableExternalVpc, nil) if err != nil { - klog.Errorf("failed to list logical switch, %v", err) + klog.Errorf("list logical switch: %v", err) return err } + klog.Infof("ls in ovn %v", lss) klog.Infof("subnet in kubernetes %v", subnetNames) for _, ls := range lss { - if ls == util.InterconnectionSwitch || - ls == util.ExternalGatewaySwitch || - ls == c.config.ExternalGatewaySwitch { + if ls.Name == util.InterconnectionSwitch || + ls.Name == util.ExternalGatewaySwitch || + ls.Name == c.config.ExternalGatewaySwitch { continue } - if s := subnetMap[ls]; s != nil && isOvnSubnet(s) { + if s := subnetMap[ls.Name]; s != nil && isOvnSubnet(s) { continue } klog.Infof("gc subnet %s", ls) - if err := c.handleDeleteLogicalSwitch(ls); err != nil { + if err := c.handleDeleteLogicalSwitch(ls.Name); err != nil { klog.Errorf("failed to gc subnet %s, %v", ls, err) return err } @@ -190,20 +184,23 @@ func (c *Controller) gcCustomLogicalRouter() error { for _, s := range vpcs { vpcNames = append(vpcNames, s.Name) } - lrs, err := c.ovnLegacyClient.ListLogicalRouter(c.config.EnableExternalVpc) + + lrs, err := c.ovnClient.ListLogicalRouter(c.config.EnableExternalVpc, nil) if err != nil { klog.Errorf("failed to list logical router, %v", err) return err } + klog.Infof("lr in ovn %v", lrs) klog.Infof("vpc in kubernetes %v", vpcNames) + for _, lr := range lrs { - if lr == util.DefaultVpc { + if lr.Name == util.DefaultVpc { continue } - if !util.IsStringIn(lr, vpcNames) { + if !util.IsStringIn(lr.Name, vpcNames) { klog.Infof("gc router %s", lr) - if err := c.deleteVpcRouter(lr); err != nil { + if err := c.deleteVpcRouter(lr.Name); err != nil { klog.Errorf("failed to delete router %s, %v", lr, err) return err } @@ -359,10 +356,11 @@ func (c *Controller) markAndCleanLSP() error { } klog.Infof("gc logical switch port %s", lsp.Name) - if err := c.ovnLegacyClient.DeleteLogicalSwitchPort(lsp.Name); err != nil { + if err := c.ovnClient.DeleteLogicalSwitchPort(lsp.Name); err != nil { klog.Errorf("failed to delete lsp %s, %v", lsp, err) return err } + if err := c.config.KubeOvnClient.KubeovnV1().IPs().Delete(context.Background(), lsp.Name, metav1.DeleteOptions{}); err != nil { if !k8serrors.IsNotFound(err) { klog.Errorf("failed to delete ip %s, %v", lsp.Name, err) @@ -842,3 +840,13 @@ func (c *Controller) gcVpcDns() error { } return nil } + +func logicalRouterPortFilter(exceptPeerPorts map[string]struct{}) func(lrp *ovnnb.LogicalRouterPort) bool { + return func(lrp *ovnnb.LogicalRouterPort) bool { + if _, ok := exceptPeerPorts[lrp.Name]; ok { + return false // ignore except lrp + } + + return lrp.Peer != nil && len(*lrp.Peer) != 0 + } +} diff --git a/pkg/controller/gc_test.go b/pkg/controller/gc_test.go new file mode 100644 index 00000000000..99ee17622eb --- /dev/null +++ b/pkg/controller/gc_test.go @@ -0,0 +1,48 @@ +package controller + +import ( + "fmt" + "testing" + + "github.com/kubeovn/kube-ovn/pkg/ovsdb/ovnnb" + "github.com/stretchr/testify/require" +) + +func newLogicalRouterPort(lrName, lrpName, mac string, networks []string) *ovnnb.LogicalRouterPort { + return &ovnnb.LogicalRouterPort{ + Name: lrpName, + MAC: mac, + Networks: networks, + ExternalIDs: map[string]string{ + "lr": lrName, + }, + } +} + +func Test_logicalRouterPortFilter(t *testing.T) { + t.Parallel() + + exceptPeerPorts := map[string]struct{}{ + "except-lrp-0": {}, + "except-lrp-1": {}, + } + + lrpNames := []string{"other-0", "other-1", "other-2", "except-lrp-0", "except-lrp-1"} + lrps := make([]*ovnnb.LogicalRouterPort, 0) + for _, lrpName := range lrpNames { + lrp := newLogicalRouterPort("", lrpName, "", nil) + peer := fmt.Sprintf("%s-peer", lrpName) + lrp.Peer = &peer + lrps = append(lrps, lrp) + } + + filterFunc := logicalRouterPortFilter(exceptPeerPorts) + + for _, lrp := range lrps { + if _, ok := exceptPeerPorts[lrp.Name]; ok { + require.False(t, filterFunc(lrp)) + } else { + require.True(t, filterFunc(lrp)) + } + } +} diff --git a/pkg/controller/init.go b/pkg/controller/init.go index 9a1a74ae9a8..e72bc0a73be 100644 --- a/pkg/controller/init.go +++ b/pkg/controller/init.go @@ -197,17 +197,7 @@ func (c *Controller) initNodeSwitch() error { // InitClusterRouter init cluster router to connect different logical switches func (c *Controller) initClusterRouter() error { - lrs, err := c.ovnLegacyClient.ListLogicalRouter(c.config.EnableExternalVpc) - if err != nil { - return err - } - klog.Infof("exists routers: %v", lrs) - for _, r := range lrs { - if c.config.ClusterRouter == r { - return nil - } - } - return c.ovnLegacyClient.CreateLogicalRouter(c.config.ClusterRouter) + return c.ovnClient.CreateLogicalRouter(c.config.ClusterRouter) } func (c *Controller) initLB(name, protocol string, sessionAffinity bool) error { @@ -798,8 +788,8 @@ func (c *Controller) initAppendLspExternalIds(portName string, pod *v1.Pod) erro externalIDs["pod"] = fmt.Sprintf("%s/%s", pod.Namespace, pod.Name) } - if err := c.ovnLegacyClient.SetLspExternalIds(portName, externalIDs); err != nil { - klog.Errorf("failed to set lsp external_ids for port %s: %v", portName, err) + if err := c.ovnClient.SetLogicalSwitchPortExternalIds(portName, externalIDs); err != nil { + klog.Errorf("set lsp external_ids for logical switch port %s: %v", portName, err) return err } diff --git a/pkg/controller/inspection.go b/pkg/controller/inspection.go index 4cb78200cf8..d16501c3fc6 100644 --- a/pkg/controller/inspection.go +++ b/pkg/controller/inspection.go @@ -21,11 +21,7 @@ func (c *Controller) inspectPod() error { klog.Errorf("failed to list ip, %v", err) return err } - lsps, err := c.ovnLegacyClient.ListLogicalSwitchPort(c.config.EnableExternalVpc) - if err != nil { - klog.Errorf("failed to list logical switch port, %v", err) - return err - } + for _, oriPod := range pods { pod := oriPod.DeepCopy() if pod.Spec.HostNetwork { @@ -45,14 +41,12 @@ func (c *Controller) inspectPod() error { for _, podNet := range filterSubnets(pod, podNets) { if podNet.Type != providerTypeIPAM { portName := ovs.PodNameToPortName(podName, pod.Namespace, podNet.ProviderName) - isLspExist := false - for _, lsp := range lsps { - if portName == lsp { - isLspExist = true - break - } + exists, err := c.ovnClient.LogicalSwitchPortExists(portName) + if err != nil { + return err } - if !isLspExist { + + if !exists { // pod exists but not lsp delete(pod.Annotations, fmt.Sprintf(util.AllocatedAnnotationTemplate, podNet.ProviderName)) delete(pod.Annotations, fmt.Sprintf(util.RoutedAnnotationTemplate, podNet.ProviderName)) patch, err := util.GenerateStrategicMergePatchPayload(oriPod, pod) diff --git a/pkg/controller/node.go b/pkg/controller/node.go index 831f7832d35..c4d0c48e411 100644 --- a/pkg/controller/node.go +++ b/pkg/controller/node.go @@ -253,7 +253,7 @@ func (c *Controller) handleAddNode(key string) error { } ipStr := util.GetStringIP(v4IP, v6IP) - if err := c.ovnLegacyClient.CreatePort(c.config.NodeSwitch, portName, ipStr, mac, "", "", false, "", "", false, false, nil, false); err != nil { + if err := c.ovnClient.CreateBareLogicalSwitchPort(c.config.NodeSwitch, portName, ipStr, mac); err != nil { return err } @@ -446,7 +446,7 @@ func (c *Controller) handleNodeAnnotationsForProviderNetworks(node *v1.Node) err func (c *Controller) handleDeleteNode(key string) error { portName := fmt.Sprintf("node-%s", key) klog.Infof("delete logical switch port %s", portName) - if err := c.ovnLegacyClient.DeleteLogicalSwitchPort(portName); err != nil { + if err := c.ovnClient.DeleteLogicalSwitchPort(portName); err != nil { klog.Errorf("failed to delete node switch port node-%s: %v", key, err) return err } diff --git a/pkg/controller/ovn-ic.go b/pkg/controller/ovn-ic.go index ad620f4c6c0..dba645b3fc3 100644 --- a/pkg/controller/ovn-ic.go +++ b/pkg/controller/ovn-ic.go @@ -167,8 +167,10 @@ func (c *Controller) removeInterConnection(azName string) error { } if azName != "" { - if err := c.ovnLegacyClient.DeleteICLogicalRouterPort(azName); err != nil { - klog.Errorf("failed to delete ovn-ic lrp, %v", err) + lspName := fmt.Sprintf("ts-%s", azName) + lrpName := fmt.Sprintf("%s-ts", azName) + if err := c.ovnClient.RemoveLogicalPatchPort(lspName, lrpName); err != nil { + klog.Errorf("delete ovn-ic logical port %s and %s: %v", lspName, lrpName, err) return err } } @@ -188,11 +190,12 @@ func (c *Controller) establishInterConnection(config map[string]string) error { } tsPort := fmt.Sprintf("ts-%s", config["az-name"]) - exist, err := c.ovnLegacyClient.LogicalSwitchPortExists(tsPort) + exist, err := c.ovnClient.LogicalSwitchPortExists(tsPort) if err != nil { klog.Errorf("failed to list logical switch ports, %v", err) return err } + if exist { klog.Infof("ts port %s already exists", tsPort) return nil @@ -256,7 +259,8 @@ func (c *Controller) establishInterConnection(config map[string]string) error { return err } - if err := c.ovnLegacyClient.CreateICLogicalRouterPort(config["az-name"], util.GenerateMac(), subnet, chassises); err != nil { + lrpName := fmt.Sprintf("%s-ts", config["az-name"]) + if err := c.ovnClient.CreateLogicalPatchPort(util.InterconnectionSwitch, c.config.ClusterRouter, tsPort, lrpName, subnet, util.GenerateMac(), chassises...); err != nil { klog.Errorf("failed to create ovn-ic lrp %v", err) return err } @@ -267,19 +271,23 @@ func (c *Controller) establishInterConnection(config map[string]string) error { func (c *Controller) acquireLrpAddress(ts string) (string, error) { cidr, err := c.ovnLegacyClient.GetTsSubnet(ts) if err != nil { - klog.Errorf("failed to get ts subnet, %v", err) + klog.Errorf("failed to get ts subnet: %v", err) return "", err } - existAddress, err := c.ovnLegacyClient.ListRemoteLogicalSwitchPortAddress() + existAddress, err := c.listRemoteLogicalSwitchPortAddress() if err != nil { - klog.Errorf("failed to list remote port address, %v", err) + klog.Errorf("list remote port address: %v", err) return "", err } + for { random := util.GenerateRandomV4IP(cidr) - if !util.ContainsString(existAddress, random) { + + // find a free address + if _, ok := existAddress[random]; !ok { return random, nil } + klog.Infof("random ip %s already exists", random) time.Sleep(1 * time.Second) } @@ -322,19 +330,20 @@ func (c *Controller) stopOvnIC() error { func (c *Controller) waitTsReady() error { retry := 6 for retry > 0 { - exists, err := c.ovnLegacyClient.LogicalSwitchExists(util.InterconnectionSwitch, false) + ready, err := c.allSubnetReady(util.InterconnectionSwitch) if err != nil { - klog.Errorf("failed to list logical switch, %v", err) return err } - if exists { + + if ready { return nil } - klog.Info("wait for ts logical switch ready") + + klog.Info("wait for logical switch %s ready", util.InterconnectionSwitch) time.Sleep(5 * time.Second) retry = retry - 1 } - return fmt.Errorf("timeout to wait ts ready") + return fmt.Errorf("timeout to wait for logical switch %s ready", util.InterconnectionSwitch) } func (c *Controller) delLearnedRoute() error { @@ -486,3 +495,30 @@ func (c *Controller) syncOneRouteToPolicy(key, value string) { } } } + +func (c *Controller) listRemoteLogicalSwitchPortAddress() (map[string]struct{}, error) { + lsps, err := c.ovnClient.ListLogicalSwitchPorts(true, nil, func(lsp *ovnnb.LogicalSwitchPort) bool { + return lsp.Type == "remote" + }) + if err != nil { + return nil, fmt.Errorf("list remote logical switch ports: %v", err) + } + + existAddress := make(map[string]struct{}) + for _, lsp := range lsps { + if len(lsp.Addresses) == 0 { + continue + } + + addresses := lsp.Addresses[0] + + fields := strings.Fields(addresses) + if len(fields) != 2 { + continue + } + + existAddress[fields[1]] = struct{}{} + } + + return existAddress, nil +} diff --git a/pkg/controller/ovn-ic_test.go b/pkg/controller/ovn-ic_test.go index b0b429f8999..ab467e93a9d 100644 --- a/pkg/controller/ovn-ic_test.go +++ b/pkg/controller/ovn-ic_test.go @@ -1 +1,42 @@ package controller + +import ( + "testing" + + "github.com/golang/mock/gomock" + "github.com/stretchr/testify/require" + + "github.com/kubeovn/kube-ovn/pkg/ovsdb/ovnnb" +) + +func Test_listRemoteLogicalSwitchPortAddress(t *testing.T) { + t.Parallel() + + fakeController := newFakeController(t) + ctrl := fakeController.fakeController + mockOvnClient := fakeController.mockOvnClient + + mockLsp := func(addresses []string) ovnnb.LogicalSwitchPort { + return ovnnb.LogicalSwitchPort{ + Addresses: addresses, + } + } + + lsps := []ovnnb.LogicalSwitchPort{ + mockLsp([]string{"00:00:00:53:21:6F 10.244.0.17 fc00::af4:11"}), + mockLsp([]string{"00:00:00:53:21:6F 10.244.0.18"}), + mockLsp([]string{"00:00:00:53:21:6E 10.244.0.10"}), + mockLsp([]string{"00:00:00:53:21:6F"}), + mockLsp([]string{""}), + mockLsp([]string{}), + } + + mockOvnClient.EXPECT().ListLogicalSwitchPorts(gomock.Any(), gomock.Any(), gomock.Any()).Return(lsps, nil) + + addresses, err := ctrl.listRemoteLogicalSwitchPortAddress() + require.NoError(t, err) + require.Equal(t, map[string]struct{}{ + "10.244.0.10": {}, + "10.244.0.18": {}, + }, addresses) +} diff --git a/pkg/controller/ovn_eip.go b/pkg/controller/ovn_eip.go index ffae5c7f129..982847deda7 100644 --- a/pkg/controller/ovn_eip.go +++ b/pkg/controller/ovn_eip.go @@ -240,10 +240,11 @@ func (c *Controller) handleAddOvnEip(key string) error { if cachedEip.Spec.Type == util.NodeExtGwUsingEip { mergedIp := util.GetStringIP(v4ip, v6ip) - if err := c.ovnLegacyClient.CreatePort(subnet.Name, portName, mergedIp, mac, "", "", false, "", "", false, false, nil, false); err != nil { + if err := c.ovnClient.CreateBareLogicalSwitchPort(subnet.Name, portName, mergedIp, mac); err != nil { klog.Error("failed to create lsp for ovn eip %s, %v", key, err) return err } + } if err = c.createOrUpdateCrdOvnEip(key, subnet.Name, v4ip, v6ip, mac, cachedEip.Spec.Type); err != nil { @@ -336,14 +337,14 @@ func (c *Controller) handleDelOvnEip(key string) error { return err } if cachedEip.Spec.Type == util.NodeExtGwUsingEip { - if err := c.ovnLegacyClient.DeleteLogicalSwitchPort(cachedEip.Name); err != nil { + if err := c.ovnClient.DeleteLogicalSwitchPort(cachedEip.Name); err != nil { klog.Errorf("failed to delete lsp %s, %v", cachedEip.Name, err) return err } } if cachedEip.Spec.Type == util.LrpUsingEip { - if err := c.ovnLegacyClient.DeleteLogicalRouterPort(key); err != nil { + if err := c.ovnClient.DeleteLogicalRouterPort(key); err != nil { klog.Errorf("failed to delete lrp %s, %v", key, err) return err } diff --git a/pkg/controller/pod.go b/pkg/controller/pod.go index 055ddd7964a..e23a6f30487 100644 --- a/pkg/controller/pod.go +++ b/pkg/controller/pod.go @@ -571,7 +571,7 @@ func (c *Controller) changeVMSubnet(vmName, namespace, providerName, subnetName for _, port := range ports { // when lsp is deleted, the port of pod is deleted from any port-group automatically. klog.Infof("gc logical switch port %s", port.Name) - if err := c.ovnLegacyClient.DeleteLogicalSwitchPort(port.Name); err != nil { + if err := c.ovnClient.DeleteLogicalSwitchPort(port.Name); err != nil { klog.Errorf("failed to delete lsp %s, %v", port.Name, err) return err } @@ -693,9 +693,9 @@ func (c *Controller) handleAddPod(key string) error { DHCPv6OptionsUUID: subnet.Status.DHCPv6OptionsUUID, } - hasUnknown := pod.Annotations[fmt.Sprintf(util.Layer2ForwardAnnotationTemplate, podNet.ProviderName)] == "true" - if err := c.ovnLegacyClient.CreatePort(subnet.Name, portName, ipStr, mac, podName, pod.Namespace, portSecurity, securityGroupAnnotation, vips, podNet.AllowLiveMigration, podNet.Subnet.Spec.EnableDHCP, dhcpOptions, hasUnknown); err != nil { + if err := c.ovnClient.CreateLogicalSwitchPort(subnet.Name, portName, ipStr, mac, podName, pod.Namespace, portSecurity, securityGroupAnnotation, vips, podNet.Subnet.Spec.EnableDHCP, dhcpOptions, subnet.Spec.Vpc); err != nil { c.recorder.Eventf(pod, v1.EventTypeWarning, "CreateOVNPortFailed", err.Error()) + klog.Errorf("%v", err) return err } @@ -818,7 +818,7 @@ func (c *Controller) handleDeletePod(pod *v1.Pod) error { for _, port := range ports { // when lsp is deleted, the port of pod is deleted from any port-group automatically. klog.Infof("gc logical switch port %s", port.Name) - if err := c.ovnLegacyClient.DeleteLogicalSwitchPort(port.Name); err != nil { + if err := c.ovnClient.DeleteLogicalSwitchPort(port.Name); err != nil { klog.Errorf("failed to delete lsp %s, %v", port.Name, err) return err } @@ -884,10 +884,12 @@ func (c *Controller) handleUpdatePodSecurity(key string) error { mac := pod.Annotations[fmt.Sprintf(util.MacAddressAnnotationTemplate, podNet.ProviderName)] ipStr := pod.Annotations[fmt.Sprintf(util.IpAddressAnnotationTemplate, podNet.ProviderName)] vips := pod.Annotations[fmt.Sprintf(util.PortVipAnnotationTemplate, podNet.ProviderName)] - if err = c.ovnLegacyClient.SetPortSecurity(portSecurity, podNet.Subnet.Name, ovs.PodNameToPortName(podName, namespace, podNet.ProviderName), mac, ipStr, vips); err != nil { - klog.Errorf("setPortSecurity failed. %v", err) + + if err = c.ovnClient.SetLogicalSwitchPortSecurity(portSecurity, ovs.PodNameToPortName(podName, namespace, podNet.ProviderName), mac, ipStr, vips); err != nil { + klog.Errorf("set logical switch port security: %v", err) return err } + c.syncVirtualPortsQueue.Add(podNet.Subnet.Name) var securityGroups string @@ -1572,53 +1574,6 @@ func appendCheckPodToDel(c *Controller, pod *v1.Pod, ownerRefName, ownerRefKind return false, nil } -// syncVmLiveMigrationPort set ip address to lsp after live migration -func (c *Controller) syncVmLiveMigrationPort() { - subnets, err := c.subnetsLister.List(labels.Everything()) - if err != nil { - klog.Errorf("list get subnet failed, %v", err) - return - } - for _, subnet := range subnets { - // lists pods with the 'liveMigration' flag - ports, err := c.ovnLegacyClient.ListLogicalEntity("logical_switch_port", - fmt.Sprintf("external_ids:ls=%s", subnet.Name), - "external_ids:liveMigration=1") - if err != nil { - klog.Errorf("list logical_switch_port failed, %v", err) - return - } - - for _, port := range ports { - addr, err := c.ipsLister.Get(port) - if err != nil { - klog.Errorf("get port ip failed, %v", err) - return - } - // lists pods with the same IP address - vmLsps, err := c.ovnLegacyClient.ListLogicalEntity("logical_switch_port", - fmt.Sprintf("external_ids:ls=%s", subnet.Name), - fmt.Sprintf("external_ids:ip=\"%s\"", strings.ReplaceAll(addr.Spec.IPAddress, ",", "/"))) - if err != nil { - klog.Errorf("list logical_switch_port failed, %v", err) - return - } - - // reset addresses after live Migration - if len(vmLsps) == 1 { - if err = c.ovnLegacyClient.SetPortAddress(port, addr.Spec.MacAddress, addr.Spec.IPAddress); err != nil { - klog.Errorf("set port addresses failed, %v", err) - return - } - if err = c.ovnLegacyClient.SetPortExternalIds(port, "liveMigration", "0"); err != nil { - klog.Errorf("set port externalIds failed, %v", err) - return - } - } - } - } -} - func isVmPod(pod *v1.Pod) (bool, string) { for _, owner := range pod.OwnerReferences { // The name of vmi is consistent with vm's name. diff --git a/pkg/controller/security_group.go b/pkg/controller/security_group.go index 799428cf755..efc7542c015 100644 --- a/pkg/controller/security_group.go +++ b/pkg/controller/security_group.go @@ -470,17 +470,19 @@ func (c *Controller) reconcilePortSg(portName, securityGroups string) error { if util.ContainsString(newSgList, sgName) { needAssociated = "true" } - if err = c.ovnLegacyClient.SetPortExternalIds(portName, fmt.Sprintf("associated_sg_%s", sgName), needAssociated); err != nil { - klog.Errorf("set port '%s' external_ids failed, %v", portName, err) + + if err = c.ovnClient.SetLogicalSwitchPortExternalIds(portName, map[string]string{fmt.Sprintf("associated_sg_%s", sgName): needAssociated}); err != nil { + klog.Errorf("set logical switch port %s external_ids: %v", portName, err) return err } c.syncSgPortsQueue.Add(sgName) } - if err = c.ovnLegacyClient.SetPortExternalIds(portName, "security_groups", strings.ReplaceAll(securityGroups, ",", "/")); err != nil { - klog.Errorf("set port '%s' external_ids failed, %v", portName, err) + if err = c.ovnClient.SetLogicalSwitchPortExternalIds(portName, map[string]string{"security_groups": strings.ReplaceAll(securityGroups, ",", "/")}); err != nil { + klog.Errorf("set logical switch port %s external_ids: %v", portName, err) return err } + return nil } diff --git a/pkg/controller/security_group_test.go b/pkg/controller/security_group_test.go index a2fd614378b..a4516c238cd 100644 --- a/pkg/controller/security_group_test.go +++ b/pkg/controller/security_group_test.go @@ -6,7 +6,6 @@ import ( "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" - mockovs "github.com/kubeovn/kube-ovn/mocks/pkg/ovs" "github.com/kubeovn/kube-ovn/pkg/ovs" "github.com/kubeovn/kube-ovn/pkg/ovsdb/ovnnb" ) @@ -43,11 +42,9 @@ func Test_getPortSg(t *testing.T) { func Test_securityGroupALLNotExist(t *testing.T) { t.Parallel() - mockCtrl := gomock.NewController(t) - mockOvnClient := mockovs.NewMockOvnClient(mockCtrl) - ctrl := &Controller{ - ovnClient: mockOvnClient, - } + fakeController := newFakeController(t) + ctrl := fakeController.fakeController + mockOvnClient := fakeController.mockOvnClient sgName := "sg" pgName := ovs.GetSgPortGroupName(sgName) diff --git a/pkg/controller/subnet.go b/pkg/controller/subnet.go index b00f8baa7f8..67a59352e8f 100644 --- a/pkg/controller/subnet.go +++ b/pkg/controller/subnet.go @@ -12,6 +12,7 @@ import ( kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1" "github.com/kubeovn/kube-ovn/pkg/ipam" "github.com/kubeovn/kube-ovn/pkg/ovs" + "github.com/kubeovn/kube-ovn/pkg/ovsdb/ovnnb" "github.com/kubeovn/kube-ovn/pkg/util" v1 "k8s.io/api/core/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" @@ -648,12 +649,6 @@ func (c *Controller) handleAddOrUpdateSubnet(key string) error { } } - exist, err := c.ovnLegacyClient.LogicalSwitchExists(subnet.Name, c.config.EnableExternalVpc) - if err != nil { - klog.Errorf("failed to list logical switch, %v", err) - c.patchSubnetStatus(subnet, "ListLogicalSwitchFailed", err.Error()) - return err - } needRouter := subnet.Spec.Vlan == "" || subnet.Spec.LogicalGateway || (subnet.Status.U2OInterconnectionIP != "" && subnet.Spec.U2OInterconnection) // 1. overlay subnet, should add lrp, lrp ip is subnet gw @@ -661,40 +656,19 @@ func (c *Controller) handleAddOrUpdateSubnet(key string) error { randomAllocateGW := !subnet.Spec.LogicalGateway && vpc.Spec.EnableExternal && subnet.Name == c.config.ExternalGatewaySwitch // 3. underlay subnet use physical gw, vpc has eip, lrp managed in vpc process, lrp ip is random allocation, not subnet gw - if !exist { - subnet.Status.EnsureStandardConditions() - // If multiple namespace use same ls name, only first one will success - if err := c.ovnLegacyClient.CreateLogicalSwitch(subnet.Name, vpc.Status.Router, needRouter); err != nil { - c.patchSubnetStatus(subnet, "CreateLogicalSwitchFailed", err.Error()) - return err - } - if needRouter { - if err := c.reconcileRouterPortBySubnet(vpc, subnet); err != nil { - klog.Errorf("failed to connect switch %s to router %s, %v", subnet.Name, vpc.Name, err) - return err - } - } - } else { - // logical switch exists, only update other_config - if !randomAllocateGW { - gateway := subnet.Spec.Gateway - if subnet.Status.U2OInterconnectionIP != "" && subnet.Spec.U2OInterconnection { - gateway = subnet.Status.U2OInterconnectionIP - } - if err := c.ovnLegacyClient.SetLogicalSwitchConfig(subnet.Name, vpc.Status.Router, subnet.Spec.Protocol, subnet.Spec.CIDRBlock, gateway, needRouter); err != nil { - c.patchSubnetStatus(subnet, "SetLogicalSwitchConfigFailed", err.Error()) - return err - } - } - if !needRouter && !randomAllocateGW { - klog.Infof("remove connection from router %s to switch %s", vpc.Status.Router, subnet.Name) - if err := c.ovnLegacyClient.RemoveRouterPort(subnet.Name, vpc.Status.Router); err != nil { - klog.Errorf("failed to remove router port from %s, %v", subnet.Name, err) - return err - } - } + gateway := subnet.Spec.Gateway + if subnet.Status.U2OInterconnectionIP != "" && subnet.Spec.U2OInterconnection { + gateway = subnet.Status.U2OInterconnectionIP + } + + // create or update logical switch + if err := c.ovnClient.CreateLogicalSwitch(subnet.Name, vpc.Status.Router, subnet.Spec.CIDRBlock, gateway, needRouter, randomAllocateGW); err != nil { + klog.Errorf("create logical switch %s: %v", subnet.Name, err) + return err } + subnet.Status.EnsureStandardConditions() + var dhcpOptionsUUIDs *ovs.DHCPOptionsUUIDs dhcpOptionsUUIDs, err = c.ovnLegacyClient.UpdateDHCPOptions(subnet.Name, subnet.Spec.CIDRBlock, subnet.Spec.Gateway, subnet.Spec.DHCPv4Options, subnet.Spec.DHCPv6Options, subnet.Spec.EnableDHCP) if err != nil { @@ -703,8 +677,9 @@ func (c *Controller) handleAddOrUpdateSubnet(key string) error { } if needRouter { - if err := c.ovnLegacyClient.UpdateRouterPortIPv6RA(subnet.Name, vpc.Status.Router, subnet.Spec.CIDRBlock, subnet.Spec.Gateway, subnet.Spec.IPv6RAConfigs, subnet.Spec.EnableIPv6RA); err != nil { - klog.Errorf("failed to update ipv6 ra configs for router port %s-%s, %v", vpc.Status.Router, subnet.Name, err) + lrpName := fmt.Sprintf("%s-%s", vpc.Status.Router, subnet.Name) + if err := c.ovnClient.UpdateLogicalRouterPortRA(lrpName, subnet.Spec.IPv6RAConfigs, subnet.Spec.EnableIPv6RA); err != nil { + klog.Errorf("update ipv6 ra configs for logical router port %s, %v", lrpName, err) return err } } @@ -812,11 +787,13 @@ func (c *Controller) handleDeleteRoute(subnet *kubeovnv1.Subnet) error { func (c *Controller) handleDeleteLogicalSwitch(key string) (err error) { c.ipam.DeleteSubnet(key) - exist, err := c.ovnLegacyClient.LogicalSwitchExists(key, c.config.EnableExternalVpc) + exist, err := c.ovnClient.LogicalSwitchExists(key) if err != nil { - klog.Errorf("failed to list logical switch, %v", err) + klog.Errorf("check logical switch %s exist: %v", key, err) return err } + + // not found, skip if !exist { return nil } @@ -831,8 +808,8 @@ func (c *Controller) handleDeleteLogicalSwitch(key string) (err error) { return err } - if err = c.ovnLegacyClient.DeleteLogicalSwitch(key); err != nil { - klog.Errorf("failed to delete logical switch %s %v", key, err) + if err = c.ovnClient.DeleteLogicalSwitch(key); err != nil { + klog.Errorf("delete logical switch %s: %v", key, err) return err } @@ -884,24 +861,24 @@ func (c *Controller) handleDeleteSubnet(subnet *kubeovnv1.Subnet) error { klog.Errorf("failed to delete logical switch %s %v", subnet.Name, err) return err } + + var router string vpc, err := c.vpcsLister.Get(subnet.Spec.Vpc) - if err == nil && vpc.Status.Router != "" { - klog.Infof("remove connection from router %s to switch %s", vpc.Status.Router, subnet.Name) - if err = c.ovnLegacyClient.RemoveRouterPort(subnet.Name, vpc.Status.Router); err != nil { - klog.Errorf("failed to delete router port %s %v", subnet.Name, err) + if err != nil { + if !k8serrors.IsNotFound(err) { + klog.Errorf("get vpc %s: %v", vpc.Name, err) return err } + router = util.DefaultVpc } else { - if k8serrors.IsNotFound(err) { - klog.Infof("remove connection from router %s to switch %s", util.DefaultVpc, subnet.Name) - if err = c.ovnLegacyClient.RemoveRouterPort(subnet.Name, util.DefaultVpc); err != nil { - klog.Errorf("failed to delete router port %s %v", subnet.Name, err) - return err - } - } else { - klog.Errorf("failed to get vpc, %v", err) - return err - } + router = vpc.Status.Router + } + + lspName := fmt.Sprintf("%s-%s", subnet.Name, router) + lrpName := fmt.Sprintf("%s-%s", router, subnet.Name) + if err = c.ovnClient.RemoveLogicalPatchPort(lspName, lrpName); err != nil { + klog.Errorf("delete router port %s and %s:%v", lspName, lrpName, err) + return err } vlans, err := c.vlansLister.List(labels.Everything()) @@ -968,45 +945,57 @@ func (c *Controller) reconcileSubnet(subnet *kubeovnv1.Subnet) error { } func (c *Controller) reconcileVips(subnet *kubeovnv1.Subnet) error { - // 1. get all vip port - results, err := c.ovnLegacyClient.CustomFindEntity("logical_switch_port", []string{"name", "options"}, "type=virtual", fmt.Sprintf("external_ids:ls=%s", subnet.Name)) + /* get all virtual port belongs to this logical switch */ + lsps, err := c.ovnClient.ListLogicalSwitchPorts(true, map[string]string{logicalSwitchKey: subnet.Name}, func(lsp *ovnnb.LogicalSwitchPort) bool { + return lsp.Type == "virtual" + }) + if err != nil { - klog.Errorf("failed to find virtual port, %v", err) + klog.Errorf("failed to find virtual port for subnet %s: %v", subnet.Name, err) return err } - // 2. remove no need port - var existVips []string - for _, ret := range results { - options := ret["options"] - for _, value := range options { - if !strings.HasPrefix(value, "virtual-ip=") { - continue - } - vip := strings.TrimPrefix(value, "virtual-ip=") - if vip == "" || net.ParseIP(vip) == nil { - continue - } - if !util.ContainsString(subnet.Spec.Vips, vip) { - klog.Infof("delete logical switch port %s", ret["name"][0]) - if err = c.ovnLegacyClient.DeleteLogicalSwitchPort(ret["name"][0]); err != nil { - klog.Errorf("failed to delete virtual port, %v", err) - return err - } - } else { - existVips = append(existVips, vip) - } + /* filter all invaild virtual port */ + existVips := make(map[string]string) // key is vip, value is port name + for _, lsp := range lsps { + vip, ok := lsp.Options["virtual-ip"] + if !ok { + continue // ingnore vip which is empty + } + + if net.ParseIP(vip) == nil { + continue // ingnore invalid vip } + + existVips[vip] = lsp.Name } - // 3. create new port - newVips := util.DiffStringSlice(existVips, subnet.Spec.Vips) - for _, vip := range newVips { - if err = c.ovnLegacyClient.CreateVirtualPort(subnet.Name, vip); err != nil { - klog.Errorf("failed to create virtual port, %v", err) + /* filter virtual port to be added and old virtual port to be deleted */ + var newVips []string + for _, vip := range subnet.Spec.Vips { + if _, ok := existVips[vip]; !ok { + // new virtual port to be added + newVips = append(newVips, vip) + } else { + // delete old virtual port that do not need to be deleted + delete(existVips, vip) + } + } + + // delete old virtual ports + for _, lspName := range existVips { + if err = c.ovnClient.DeleteLogicalSwitchPort(lspName); err != nil { + klog.Errorf("delete virtual port %s lspName from logical switch %s: %v", lspName, subnet.Name, err) return err } } + + // add new virtual port + if err = c.ovnClient.CreateVirtualLogicalSwitchPorts(subnet.Name, newVips...); err != nil { + klog.Errorf("create virtual port with vips %v from logical switch %s: %v", newVips, subnet.Name, err) + return err + } + c.syncVirtualPortsQueue.Add(subnet.Name) return nil } @@ -1025,41 +1014,46 @@ func (c *Controller) syncVirtualPort(key string) error { return nil } - results, err := c.ovnLegacyClient.CustomFindEntity("logical_switch_port", []string{"name", "external_ids"}, - fmt.Sprintf("external_ids:ls=%s", subnet.Name), "external_ids:attach-vips=true") + externalIDs := map[string]string{ + logicalSwitchKey: subnet.Name, + "attach-vips": "true", + } + + lsps, err := c.ovnClient.ListNormalLogicalSwitchPorts(true, externalIDs) if err != nil { - klog.Errorf("failed to list logical_switch_port, %v", err) + klog.Errorf("list logical switch %s ports: %v", subnet.Name, err) return err } - vipVirtualParentsMap := map[string][]string{} - for _, ret := range results { - var associatedVips []string - for _, value := range ret["external_ids"] { - if strings.HasPrefix(value, "vips") { - vips := strings.Split(value, "=")[1] - associatedVips = strings.Split(strings.ReplaceAll(vips, " ", ""), "/") - } - } - klog.Infof("associatedVips %v", associatedVips) - for _, vip := range associatedVips { - vipVirtualParentsMap[vip] = append(vipVirtualParentsMap[vip], ret["name"][0]) - } - } for _, vip := range subnet.Spec.Vips { if !util.CIDRContainIP(subnet.Spec.CIDRBlock, vip) { klog.Errorf("vip %s is out of range to subnet %s", vip, subnet.Name) continue } + var virtualParents []string - if value, exist := vipVirtualParentsMap[vip]; exist { - virtualParents = value + for _, lsp := range lsps { + vips, ok := lsp.ExternalIDs["vips"] + if !ok { + continue // ingnore vips which is empty + } + + if strings.Contains(vips, vip) { + virtualParents = append(virtualParents, lsp.Name) + } } - if err = c.ovnLegacyClient.SetVirtualParents(subnet.Name, vip, strings.Join(virtualParents, ",")); err != nil { - klog.Errorf("failed to set vip %s virtual parents, %v", vip, err) + + // logical switch port has no vaild vip + if len(virtualParents) == 0 { + continue + } + + if err = c.ovnClient.SetLogicalSwitchPortVirtualParents(subnet.Name, strings.Join(virtualParents, ","), vip); err != nil { + klog.Errorf("set vip %s virtual parents %v: %v", vip, virtualParents, err) return err } } + return nil } @@ -1338,14 +1332,14 @@ func (c *Controller) reconcileOvnDefaultVpcRoute(subnet *kubeovnv1.Subnet) error if !subnet.Spec.LogicalGateway && subnet.Name != c.config.ExternalGatewaySwitch && !subnet.Spec.U2OInterconnection { lspName := fmt.Sprintf("%s-%s", subnet.Name, c.config.ClusterRouter) klog.Infof("delete logical switch port %s", lspName) - if err := c.ovnLegacyClient.DeleteLogicalSwitchPort(lspName); err != nil { + if err := c.ovnClient.DeleteLogicalSwitchPort(lspName); err != nil { klog.Errorf("failed to delete lsp %s-%s, %v", subnet.Name, c.config.ClusterRouter, err) return err } lrpName := fmt.Sprintf("%s-%s", c.config.ClusterRouter, subnet.Name) klog.Infof("delete logical router port %s", lrpName) - if err := c.ovnLegacyClient.DeleteLogicalRouterPort(lrpName); err != nil { - klog.Errorf("failed to delete lrp %s, %v", lrpName, err) + if err := c.ovnClient.DeleteLogicalRouterPort(lrpName); err != nil { + klog.Errorf("failed to delete lrp %s: %v", lrpName, err) return err } } @@ -1761,8 +1755,8 @@ func (c *Controller) reconcileVlan(subnet *kubeovnv1.Subnet) error { } localnetPort := ovs.GetLocalnetName(subnet.Name) - if err := c.ovnLegacyClient.CreateLocalnetPort(subnet.Name, localnetPort, vlan.Spec.Provider, vlan.Spec.ID); err != nil { - klog.Errorf("failed to create localnet port for subnet %s: %v", subnet.Name, err) + if err := c.ovnClient.CreateLocalnetLogicalSwitchPort(subnet.Name, localnetPort, vlan.Spec.Provider, vlan.Spec.ID); err != nil { + klog.Errorf("create localnet port for subnet %s: %v", subnet.Name, err) return err } diff --git a/pkg/controller/subnet_test.go b/pkg/controller/subnet_test.go new file mode 100644 index 00000000000..f328ddf1ea8 --- /dev/null +++ b/pkg/controller/subnet_test.go @@ -0,0 +1,133 @@ +package controller + +import ( + "fmt" + "testing" + + "github.com/golang/mock/gomock" + "github.com/stretchr/testify/require" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1" + "github.com/kubeovn/kube-ovn/pkg/ovsdb/ovnnb" +) + +func Test_reconcileVips(t *testing.T) { + t.Parallel() + + fakeController := newFakeController(t) + ctrl := fakeController.fakeController + mockOvnClient := fakeController.mockOvnClient + + lspNamePrefix := "reconcile-vip-lsp" + + subnet := &kubeovnv1.Subnet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "ovn-test", + }, + Spec: kubeovnv1.SubnetSpec{ + Vips: []string{"192.168.123.10", "192.168.123.11", "192.168.123.12", "192.168.123.13"}, + }, + } + + mockLsp := func(lsName, lspName, vip string) ovnnb.LogicalSwitchPort { + return ovnnb.LogicalSwitchPort{ + Name: lspName, + ExternalIDs: map[string]string{ + "ls": lsName, + }, + Options: map[string]string{ + "virtual-ip": vip, + }, + } + } + + lsps := []ovnnb.LogicalSwitchPort{ + mockLsp("", lspNamePrefix+"-0", "192.168.123.8"), + mockLsp("", lspNamePrefix+"-1", "192.168.123.9"), + mockLsp("", lspNamePrefix+"-2", "192.168.123.10"), + } + + t.Run("existent vips and new vips has intersection", func(t *testing.T) { + mockOvnClient.EXPECT().ListLogicalSwitchPorts(true, gomock.Any(), gomock.Any()).Return(lsps, nil) + mockOvnClient.EXPECT().DeleteLogicalSwitchPort(lspNamePrefix + "-0").Return(nil) + mockOvnClient.EXPECT().DeleteLogicalSwitchPort(lspNamePrefix + "-1").Return(nil) + mockOvnClient.EXPECT().CreateVirtualLogicalSwitchPorts(subnet.Name, "192.168.123.11", "192.168.123.12", "192.168.123.13").Return(nil) + + err := ctrl.reconcileVips(subnet) + require.NoError(t, err) + }) + + t.Run("existent vips is empty", func(t *testing.T) { + mockOvnClient.EXPECT().ListLogicalSwitchPorts(true, gomock.Any(), gomock.Any()).Return(nil, nil) + mockOvnClient.EXPECT().CreateVirtualLogicalSwitchPorts(subnet.Name, "192.168.123.10", "192.168.123.11", "192.168.123.12", "192.168.123.13").Return(nil) + + err := ctrl.reconcileVips(subnet) + require.NoError(t, err) + }) + + t.Run("new vips is empty", func(t *testing.T) { + subnet := &kubeovnv1.Subnet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "ovn-test", + }, + } + + mockOvnClient.EXPECT().ListLogicalSwitchPorts(true, gomock.Any(), gomock.Any()).Return(lsps, nil) + mockOvnClient.EXPECT().DeleteLogicalSwitchPort(lspNamePrefix + "-0").Return(nil) + mockOvnClient.EXPECT().DeleteLogicalSwitchPort(lspNamePrefix + "-1").Return(nil) + mockOvnClient.EXPECT().DeleteLogicalSwitchPort(lspNamePrefix + "-2").Return(nil) + mockOvnClient.EXPECT().CreateVirtualLogicalSwitchPorts(subnet.Name).Return(nil) + + err := ctrl.reconcileVips(subnet) + require.NoError(t, err) + }) +} + +func Test_syncVirtualPort(t *testing.T) { + t.Parallel() + + fakeController := newFakeController(t) + ctrl := fakeController.fakeController + fakeinformers := fakeController.fakeinformers + mockOvnClient := fakeController.mockOvnClient + + lspNamePrefix := "sync-virt-lsp" + + subnet := &kubeovnv1.Subnet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "ovn-test", + }, + Spec: kubeovnv1.SubnetSpec{ + CIDRBlock: "192.168.123.0/24", + Vips: []string{"192.168.123.10", "192.168.123.11", "192.168.123.12", "192.168.123.13"}, + }, + } + + err := fakeinformers.sbunetInformer.Informer().GetStore().Add(subnet) + require.NoError(t, err) + + mockLsp := func(lsName, lspName, vips string) ovnnb.LogicalSwitchPort { + return ovnnb.LogicalSwitchPort{ + Name: lspName, + ExternalIDs: map[string]string{ + "ls": lsName, + "vips": vips, + }, + } + } + + lsps := []ovnnb.LogicalSwitchPort{ + mockLsp("", lspNamePrefix+"-0", "192.168.123.10,192.168.123.11"), + mockLsp("", lspNamePrefix+"-1", "192.168.123.10,192.168.123.11"), + } + + mockOvnClient.EXPECT().ListNormalLogicalSwitchPorts(true, gomock.Any()).Return(lsps, nil) + virtualParents := fmt.Sprintf("%s,%s", lspNamePrefix+"-0", lspNamePrefix+"-1") + mockOvnClient.EXPECT().SetLogicalSwitchPortVirtualParents(subnet.Name, virtualParents, "192.168.123.10").Return(nil) + mockOvnClient.EXPECT().SetLogicalSwitchPortVirtualParents(subnet.Name, virtualParents, "192.168.123.11").Return(nil) + + err = ctrl.syncVirtualPort(subnet.Name) + require.NoError(t, err) +} diff --git a/pkg/controller/vlan.go b/pkg/controller/vlan.go index d738b0e689a..eac63ad0ce6 100644 --- a/pkg/controller/vlan.go +++ b/pkg/controller/vlan.go @@ -308,8 +308,8 @@ func (c *Controller) updateProviderNetworkStatusForVlanDeletion(pn *kubeovnv1.Pr func (c *Controller) setLocalnetTag(subnet string, vlanID int) error { localnetPort := ovs.GetLocalnetName(subnet) - if err := c.ovnLegacyClient.SetPortTag(localnetPort, vlanID); err != nil { - klog.Errorf("failed to set vlan tag of localnet port %s: %v", localnetPort, err) + if err := c.ovnClient.SetLogicalSwitchPortVlanTag(localnetPort, vlanID); err != nil { + klog.Errorf("set localnet port %s vlan tag %d: %v", localnetPort, vlanID, err) return err } @@ -318,9 +318,8 @@ func (c *Controller) setLocalnetTag(subnet string, vlanID int) error { func (c *Controller) delLocalnet(subnet string) error { localnetPort := ovs.GetLocalnetName(subnet) - klog.Infof("delete logical switch port %s", localnetPort) - if err := c.ovnLegacyClient.DeleteLogicalSwitchPort(localnetPort); err != nil { - klog.Errorf("failed to delete localnet port %s: %v", localnetPort, err) + if err := c.ovnClient.DeleteLogicalSwitchPort(localnetPort); err != nil { + klog.Errorf("delete localnet port %s: %v", localnetPort, err) return err } diff --git a/pkg/controller/vpc.go b/pkg/controller/vpc.go index 08a58e3af7b..2d7aea19c80 100644 --- a/pkg/controller/vpc.go +++ b/pkg/controller/vpc.go @@ -167,87 +167,6 @@ func (c *Controller) handleUpdateVpcStatus(key string) error { return nil } -func (c *Controller) reconcileRouterPorts(vpc *kubeovnv1.Vpc) error { - subnets, _, err := c.getVpcSubnets(vpc) - if err != nil { - klog.ErrorS(err, "unable to get related subnets", "vpc", vpc.Name) - return err - } - - router := vpc.Name - for _, subnetName := range subnets { - routerPortName := ovs.LogicalRouterPortName(router, subnetName) - exists, err := c.ovnClient.LogicalRouterPortExists(routerPortName) - if err != nil { - return err - } - - if !exists { - subnet, err := c.subnetsLister.Get(subnetName) - if err != nil { - if k8serrors.IsNotFound(err) { - continue - } - klog.ErrorS(err, "unable to get subnet", "subnet", subnetName) - return err - } - - if subnet.Spec.Vlan != "" && !subnet.Spec.LogicalGateway { - // skip vlan subnet which use underlay gw - // vpc connect to external vlan subnet is controlled by vpc spec enableExternal - klog.Infof("no need to connect vpc '%s' to vlan subnet %s", router, subnet.Name) - return nil - } - - gateway := subnet.Spec.Gateway - if subnet.Spec.U2OInterconnection && subnet.Status.U2OInterconnectionIP != "" { - gateway = subnet.Status.U2OInterconnectionIP - } - networks := util.GetIpAddrWithMask(gateway, subnet.Spec.CIDRBlock) - klog.Infof("add vpc lrp %s, networks %s", routerPortName, networks) - - if err := c.ovnClient.CreateLogicalRouterPort(router, routerPortName, "", strings.Split(networks, ",")); err != nil { - klog.Errorf("failed to create router port %s, err %v", routerPortName, err) - return err - } - } - } - return nil -} - -func (c *Controller) reconcileRouterPortBySubnet(vpc *kubeovnv1.Vpc, subnet *kubeovnv1.Subnet) error { - router := vpc.Name - routerPortName := ovs.LogicalRouterPortName(router, subnet.Name) - exists, err := c.ovnClient.LogicalRouterPortExists(routerPortName) - if err != nil { - return err - } - - if !exists { - subnet, err := c.subnetsLister.Get(subnet.Name) - if err != nil { - if k8serrors.IsNotFound(err) { - return nil - } - klog.Errorf("failed to get subnet %s, err %v", subnet.Name, err) - return err - } - - gateway := subnet.Spec.Gateway - if subnet.Spec.U2OInterconnection && subnet.Status.U2OInterconnectionIP != "" { - gateway = subnet.Status.U2OInterconnectionIP - } - networks := util.GetIpAddrWithMask(gateway, subnet.Spec.CIDRBlock) - klog.Infof("router port does not exist, trying to create %s with ip %s", routerPortName, networks) - - if err := c.ovnClient.CreateLogicalRouterPort(router, routerPortName, "", strings.Split(networks, ",")); err != nil { - klog.Errorf("failed to create router port %s, err %v", routerPortName, err) - return err - } - } - return nil -} - type VpcLoadBalancer struct { TcpLoadBalancer string TcpSessLoadBalancer string @@ -322,29 +241,23 @@ func (c *Controller) handleAddOrUpdateVpc(key string) error { return err } - if err := c.reconcileRouterPorts(vpc); err != nil { - klog.ErrorS(err, "unable to reconcileRouterPorts") - return err - } - var newPeers []string for _, peering := range vpc.Spec.VpcPeerings { if err = util.CheckCidrs(peering.LocalConnectIP); err != nil { klog.Errorf("invalid cidr %s", peering.LocalConnectIP) return err } + newPeers = append(newPeers, peering.RemoteVpc) - if err := c.ovnLegacyClient.CreatePeerRouterPort(vpc.Name, peering.RemoteVpc, peering.LocalConnectIP); err != nil { - klog.Errorf("failed to create peer router port for vpc %s, %v", vpc.Name, err) + if err := c.ovnClient.CreatePeerRouterPort(vpc.Name, peering.RemoteVpc, peering.LocalConnectIP); err != nil { + klog.Errorf("create peer router port for vpc %s, %v", vpc.Name, err) return err } } for _, oldPeer := range vpc.Status.VpcPeerings { if !util.ContainsString(newPeers, oldPeer) { - lrpName := fmt.Sprintf("%s-%s", vpc.Name, oldPeer) - klog.Infof("delete logical router port %s", lrpName) - if err = c.ovnLegacyClient.DeleteLogicalRouterPort(lrpName); err != nil { - klog.Errorf("failed to delete peer router port for vpc %s, %v", vpc.Name, err) + if err = c.ovnClient.DeleteLogicalRouterPort(fmt.Sprintf("%s-%s", vpc.Name, oldPeer)); err != nil { + klog.Errorf("delete peer router port for vpc %s, %v", vpc.Name, err) return err } } @@ -394,12 +307,13 @@ func (c *Controller) handleAddOrUpdateVpc(key string) error { nextHop = strings.Split(nextHop, "/")[0] } - nats, err := c.ovnLegacyClient.GetRouterNat(vpc.Name) + lr, err := c.ovnClient.GetLogicalRouter(vpc.Name, false) if err != nil { - klog.Errorf("failed to get nat for vpc %s, %v", vpc.Name, err) + klog.Errorf("failed to get logical router %s: %v", vpc.Name, err) return err } - for _, nat := range nats { + + for _, nat := range lr.Nat { logical_ip, err := c.ovnLegacyClient.GetNatIPInfo(nat) if err != nil { klog.Errorf("failed to get nat ip info for vpc %s, %v", vpc.Name, err) @@ -829,22 +743,12 @@ func (c *Controller) getVpcSubnets(vpc *kubeovnv1.Vpc) (subnets []string, defaul // createVpcRouter create router to connect logical switches in vpc func (c *Controller) createVpcRouter(lr string) error { - lrs, err := c.ovnLegacyClient.ListLogicalRouter(c.config.EnableExternalVpc) - if err != nil { - return err - } - klog.Infof("exists routers %v", lrs) - for _, r := range lrs { - if lr == r { - return nil - } - } - return c.ovnLegacyClient.CreateLogicalRouter(lr) + return c.ovnClient.CreateLogicalRouter(lr) } // deleteVpcRouter delete router to connect logical switches in vpc func (c *Controller) deleteVpcRouter(lr string) error { - return c.ovnLegacyClient.DeleteLogicalRouter(lr) + return c.ovnClient.DeleteLogicalRouter(lr) } func (c *Controller) handleAddVpcExternal(key string) error { @@ -900,11 +804,16 @@ func (c *Controller) handleAddVpcExternal(key string) error { klog.Errorf("failed to get gateway chassis, %v", err) return err } + v4ipCidr := util.GetIpAddrWithMask(v4ip, cachedSubnet.Spec.CIDRBlock) - if err := c.ovnLegacyClient.ConnectRouterToExternal(c.config.ExternalGatewaySwitch, key, v4ipCidr, mac, chassises); err != nil { - klog.Errorf("failed to connect router '%s' to external, %v", key, err) + lspName := fmt.Sprintf("%s-%s", c.config.ExternalGatewaySwitch, key) + lrpName := fmt.Sprintf("%s-%s", key, c.config.ExternalGatewaySwitch) + + if err := c.ovnClient.CreateLogicalPatchPort(c.config.ExternalGatewaySwitch, key, lspName, lrpName, v4ipCidr, mac, chassises...); err != nil { + klog.Errorf("failed to connect router '%s' to external: %v", key, err) return err } + cachedVpc, err := c.vpcsLister.Get(key) if err != nil { if k8serrors.IsNotFound(err) { @@ -979,12 +888,16 @@ func (c *Controller) handleDelVpcExternal(key string) error { } return err } - lrpEipName := fmt.Sprintf("%s-%s", key, c.config.ExternalGatewaySwitch) - klog.V(3).Infof("delete vpc lrp %s", lrpEipName) - if err := c.ovnLegacyClient.DisconnectRouterToExternal(c.config.ExternalGatewaySwitch, key); err != nil { + + lspName := fmt.Sprintf("%s-%s", c.config.ExternalGatewaySwitch, key) + lrpName := fmt.Sprintf("%s-%s", key, c.config.ExternalGatewaySwitch) + klog.V(3).Infof("delete vpc lrp %s", lrpName) + + if err := c.ovnClient.RemoveLogicalPatchPort(lspName, lrpName); err != nil { klog.Errorf("failed to disconnect router '%s' to external, %v", key, err) return err } + vpc := cachedVpc.DeepCopy() vpc.Status.EnableExternal = cachedVpc.Spec.EnableExternal bytes, err := vpc.Status.Bytes() @@ -995,7 +908,7 @@ func (c *Controller) handleDelVpcExternal(key string) error { vpc.Name, types.MergePatchType, bytes, metav1.PatchOptions{}, "status"); err != nil { return err } - cachedEip, err := c.ovnEipsLister.Get(lrpEipName) + cachedEip, err := c.ovnEipsLister.Get(lrpName) if err != nil { if k8serrors.IsNotFound(err) { return nil @@ -1006,15 +919,15 @@ func (c *Controller) handleDelVpcExternal(key string) error { klog.Errorf("failed to del finalizer for ovn eip, %v", err) return err } - if err = c.config.KubeOvnClient.KubeovnV1().OvnEips().Delete(context.Background(), lrpEipName, metav1.DeleteOptions{}); err != nil { + if err = c.config.KubeOvnClient.KubeovnV1().OvnEips().Delete(context.Background(), lrpName, metav1.DeleteOptions{}); err != nil { if !k8serrors.IsNotFound(err) { - klog.Errorf("failed to delete ovn eip %s, %v", lrpEipName, err) + klog.Errorf("failed to delete ovn eip %s, %v", lrpName, err) return err } } // del all vpc bfds - if err := c.ovnLegacyClient.DeleteBfd(lrpEipName, ""); err != nil { + if err := c.ovnLegacyClient.DeleteBfd(lrpName, ""); err != nil { klog.Error(err) return err } diff --git a/pkg/ovs/ovn-nbctl-legacy.go b/pkg/ovs/ovn-nbctl-legacy.go index 275ef555a40..1f3c551e3e8 100644 --- a/pkg/ovs/ovn-nbctl-legacy.go +++ b/pkg/ovs/ovn-nbctl-legacy.go @@ -74,400 +74,6 @@ func (c LegacyClient) GetVersion() (string, error) { return c.Version, nil } -// DeleteLogicalSwitchPort delete logical switch port in ovn -func (c LegacyClient) DeleteLogicalSwitchPort(port string) error { - klog.Infof("delete lsp %s", port) - if _, err := c.ovnNbCommand(IfExists, "lsp-del", port); err != nil { - return fmt.Errorf("failed to delete logical switch port %s, %v", port, err) - } - return nil -} - -// DeleteLogicalRouterPort delete logical switch port in ovn -func (c LegacyClient) DeleteLogicalRouterPort(port string) error { - klog.Infof("delete lrp %s", port) - if _, err := c.ovnNbCommand(IfExists, "lrp-del", port); err != nil { - return fmt.Errorf("failed to delete logical router port %s, %v", port, err) - } - return nil -} - -func (c LegacyClient) CreateICLogicalRouterPort(az, mac, subnet string, chassises []string) error { - lrpName := fmt.Sprintf("%s-ts", az) - klog.Infof("add vpc lrp %s", lrpName) - if _, err := c.ovnNbCommand(MayExist, "lrp-add", c.ClusterRouter, lrpName, mac, subnet); err != nil { - return fmt.Errorf("failed to create ovn-ic lrp, %v", err) - } - if _, err := c.ovnNbCommand(MayExist, "lsp-add", util.InterconnectionSwitch, fmt.Sprintf("ts-%s", az), "--", - "lsp-set-addresses", fmt.Sprintf("ts-%s", az), "router", "--", - "lsp-set-type", fmt.Sprintf("ts-%s", az), "router", "--", - "lsp-set-options", fmt.Sprintf("ts-%s", az), fmt.Sprintf("router-port=%s-ts", az), "--", - "set", "logical_switch_port", fmt.Sprintf("ts-%s", az), fmt.Sprintf("external_ids:vendor=%s", util.CniTypeName)); err != nil { - return fmt.Errorf("failed to create ovn-ic lsp, %v", err) - } - for index, chassis := range chassises { - if _, err := c.ovnNbCommand("lrp-set-gateway-chassis", fmt.Sprintf("%s-ts", az), chassis, fmt.Sprintf("%d", 100-index)); err != nil { - return fmt.Errorf("failed to set gateway chassis, %v", err) - } - } - return nil -} - -func (c LegacyClient) DeleteICLogicalRouterPort(az string) error { - if err := c.DeleteLogicalRouterPort(fmt.Sprintf("%s-ts", az)); err != nil { - return fmt.Errorf("failed to delete ovn-ic logical router port: %v", err) - } - if err := c.DeleteLogicalSwitchPort(fmt.Sprintf("ts-%s", az)); err != nil { - return fmt.Errorf("failed to delete ovn-ic logical switch port: %v", err) - } - return nil -} - -func (c LegacyClient) SetPortAddress(port, mac, ip string) error { - rets, err := c.ListLogicalEntity("logical_switch_port", fmt.Sprintf("name=%s", port)) - if err != nil { - return fmt.Errorf("failed to find port %s: %v", port, err) - } - if len(rets) == 0 { - return nil - } - - var addresses []string - addresses = append(addresses, mac) - addresses = append(addresses, strings.Split(ip, ",")...) - if _, err := c.ovnNbCommand("lsp-set-addresses", port, strings.Join(addresses, " ")); err != nil { - klog.Errorf("set port %s addresses failed, %v", port, err) - return err - } - return nil -} - -func (c LegacyClient) SetPortExternalIds(port, key, value string) error { - rets, err := c.ListLogicalEntity("logical_switch_port", fmt.Sprintf("name=%s", port)) - if err != nil { - return fmt.Errorf("failed to find port %s: %v", port, err) - } - if len(rets) == 0 { - return nil - } - - if _, err := c.ovnNbCommand("set", "logical_switch_port", port, fmt.Sprintf("external_ids:%s=\"%s\"", key, value)); err != nil { - klog.Errorf("set port %s external_ids failed: %v", port, err) - return err - } - return nil -} - -func (c LegacyClient) SetPortSecurity(portSecurity bool, ls, port, mac, ipStr, vips string) error { - var addresses []string - ovnCommand := []string{"lsp-set-port-security", port} - if portSecurity { - addresses = append(addresses, mac) - addresses = append(addresses, strings.Split(ipStr, ",")...) - if vips != "" { - addresses = append(addresses, strings.Split(vips, ",")...) - } - ovnCommand = append(ovnCommand, strings.Join(addresses, " ")) - } - ovnCommand = append(ovnCommand, "--", "set", "logical_switch_port", port, - fmt.Sprintf("external_ids:ls=%s", ls)) - - if vips != "" { - ovnCommand = append(ovnCommand, "--", "set", "logical_switch_port", port, - fmt.Sprintf("external_ids:vips=%s", strings.ReplaceAll(vips, ",", "/")), "external_ids:attach-vips=true") - - } else { - ovnCommand = append(ovnCommand, "--", "remove", "logical_switch_port", port, "external_ids", "attach-vips", "vips") - } - - if _, err := c.ovnNbCommand(ovnCommand...); err != nil { - klog.Errorf("set port %s security failed: %v", port, err) - return err - } - return nil -} - -// CreateVirtualPort create virtual type logical switch port in ovn -func (c LegacyClient) CreateVirtualPort(ls, ip string) error { - portName := fmt.Sprintf("%s-vip-%s", ls, ip) - if _, err := c.ovnNbCommand(MayExist, "lsp-add", ls, portName, - "--", "set", "logical_switch_port", portName, "type=virtual", - fmt.Sprintf("options:virtual-ip=\"%s\"", ip), - fmt.Sprintf("external_ids:vendor=%s", util.CniTypeName), - fmt.Sprintf("external_ids:ls=%s", ls)); err != nil { - klog.Errorf("create virtual port %s failed: %v", portName, err) - return err - } - return nil -} - -func (c LegacyClient) SetVirtualParents(ls, ip, parents string) error { - klog.Infof("set virtual parents: ls='%s', vip='%s', parents='%s'", ls, ip, parents) - portName := fmt.Sprintf("%s-vip-%s", ls, ip) - var cmdArg []string - if parents != "" { - cmdArg = append(cmdArg, "set", "logical_switch_port", portName, fmt.Sprintf("options:virtual-parents=%s", parents)) - } else { - cmdArg = append(cmdArg, "remove", "logical_switch_port", portName, "options", "virtual-parents") - } - if _, err := c.ovnNbCommand(cmdArg...); err != nil { - klog.Errorf("set vip %s virtual parents failed: %v", ip, err) - return err - } - return nil -} - -func (c LegacyClient) ListVirtualPort(ls string) ([]string, error) { - cmdArg := []string{"--format=csv", "--data=bare", "--no-heading", "--columns=name", "find", "logical_switch_port", "type=virtual", fmt.Sprintf("external_ids:ls=%s", ls)} - output, err := c.ovnNbCommand(cmdArg...) - if err != nil { - klog.Errorf("failed to list logical switch port, %v", err) - return nil, err - } - lines := strings.Split(output, "\n") - result := make([]string, 0, len(lines)) - for _, l := range lines { - if len(strings.TrimSpace(l)) == 0 { - continue - } - result = append(result, strings.TrimSpace(l)) - } - return result, nil -} - -// CreatePort create logical switch port in ovn -func (c LegacyClient) CreatePort(ls, port, ip, mac, pod, namespace string, portSecurity bool, securityGroups string, vips string, liveMigration bool, enableDHCP bool, dhcpOptions *DHCPOptionsUUIDs, hasUnknown bool) error { - var ovnCommand []string - var addresses []string - addresses = append(addresses, mac) - addresses = append(addresses, strings.Split(ip, ",")...) - - ovnCommand = []string{MayExist, "lsp-add", ls, port} - isAddrConflict := false - - // add external_id info - ovnCommand = append(ovnCommand, - "--", "set", "logical_switch_port", port, fmt.Sprintf("external_ids:ls=%s", ls), - "--", "set", "logical_switch_port", port, fmt.Sprintf("external_ids:ip=%s", strings.ReplaceAll(ip, ",", "/"))) - - if liveMigration { - ports, err := c.ListLogicalEntity("logical_switch_port", - fmt.Sprintf("external_ids:ls=%s", ls), - fmt.Sprintf("external_ids:ip=\"%s\"", strings.ReplaceAll(ip, ",", "/"))) - if err != nil { - klog.Errorf("list logical entity failed: %v", err) - return err - } - if len(ports) > 0 { - isAddrConflict = true - } - } - - if isAddrConflict { - // only set mac, and set flag 'liveMigration' - ovnCommand = append(ovnCommand, "--", "lsp-set-addresses", port, mac, "--", - "set", "logical_switch_port", port, "external_ids:liveMigration=1") - } else { - // set mac and ip - ovnCommand = append(ovnCommand, - "--", "lsp-set-addresses", port, strings.Join(addresses, " ")) - - if hasUnknown { - ovnCommand = append(ovnCommand, "unknown") - } - } - - if portSecurity { - if vips != "" { - addresses = append(addresses, strings.Split(vips, ",")...) - } - ovnCommand = append(ovnCommand, - "--", "lsp-set-port-security", port, strings.Join(addresses, " ")) - - if securityGroups != "" { - sgList := strings.Split(securityGroups, ",") - ovnCommand = append(ovnCommand, - "--", "set", "logical_switch_port", port, fmt.Sprintf("external_ids:security_groups=%s", strings.ReplaceAll(securityGroups, ",", "/"))) - for _, sg := range sgList { - ovnCommand = append(ovnCommand, - "--", "set", "logical_switch_port", port, fmt.Sprintf("external_ids:associated_sg_%s=true", sg)) - } - } - } - - // set vip tag to external_id - if vips != "" { - ovnCommand = append(ovnCommand, "--", "set", "logical_switch_port", port, - fmt.Sprintf("external_ids:vips=%s", strings.ReplaceAll(vips, ",", "/")), "external_ids:attach-vips=true") - } - - if pod != "" && namespace != "" { - ovnCommand = append(ovnCommand, - "--", "set", "logical_switch_port", port, fmt.Sprintf("external_ids:pod=%s/%s", namespace, pod), fmt.Sprintf("external_ids:vendor=%s", util.CniTypeName)) - } else { - ovnCommand = append(ovnCommand, - "--", "set", "logical_switch_port", port, fmt.Sprintf("external_ids:vendor=%s", util.CniTypeName)) - } - - if enableDHCP && dhcpOptions != nil { - if len(dhcpOptions.DHCPv4OptionsUUID) != 0 { - ovnCommand = append(ovnCommand, - "--", "lsp-set-dhcpv4-options", port, dhcpOptions.DHCPv4OptionsUUID) - } - if len(dhcpOptions.DHCPv6OptionsUUID) != 0 { - ovnCommand = append(ovnCommand, - "--", "lsp-set-dhcpv6-options", port, dhcpOptions.DHCPv6OptionsUUID) - } - } - - if _, err := c.ovnNbCommand(ovnCommand...); err != nil { - klog.Errorf("create port %s failed: %v", port, err) - return err - } - return nil -} - -func (c LegacyClient) SetPortTag(name string, vlanID int) error { - output, err := c.ovnNbCommand("get", "logical_switch_port", name, "tag") - if err != nil { - klog.Errorf("failed to get tag of logical switch port %s: %v, %q", name, err, output) - return err - } - - var newTag string - if vlanID > 0 && vlanID < 4096 { - newTag = strconv.Itoa(vlanID) - } - oldTag := strings.Trim(output, "[]") - if oldTag == newTag { - return nil - } - - if oldTag != "" { - if output, err = c.ovnNbCommand("remove", "logical_switch_port", name, "tag", oldTag); err != nil { - klog.Errorf("failed to remove tag of logical switch port %s: %v, %q", name, err, output) - return err - } - } - if newTag != "" { - if output, err = c.ovnNbCommand("add", "logical_switch_port", name, "tag", newTag); err != nil { - klog.Errorf("failed to get tag of logical switch port %s: %v, %q", name, err, output) - return err - } - } - - return nil -} - -func (c LegacyClient) ListPodLogicalSwitchPorts(pod, namespace string) ([]string, error) { - output, err := c.ovnNbCommand("--format=csv", "--data=bare", "--no-heading", "--columns=name", "find", "logical_switch_port", fmt.Sprintf("external_ids:pod=%s/%s", namespace, pod)) - if err != nil { - klog.Errorf("failed to list logical switch port, %v", err) - return nil, err - } - lines := strings.Split(output, "\n") - result := make([]string, 0, len(lines)) - for _, l := range lines { - if len(strings.TrimSpace(l)) == 0 { - continue - } - result = append(result, strings.TrimSpace(l)) - } - return result, nil -} - -func (c LegacyClient) SetLogicalSwitchConfig(ls, lr, protocol, subnet, gateway string, needRouter bool) error { - klog.Infof("set logical switch: ls %s, lr %s, protocol %s, subnet %s, gw %s", ls, lr, protocol, subnet, gateway) - var err error - cidrBlocks := strings.Split(subnet, ",") - temp := strings.Split(cidrBlocks[0], "/") - if len(temp) != 2 { - klog.Errorf("cidrBlock %s is invalid", cidrBlocks[0]) - return err - } - mask := temp[1] - - var cmd []string - var networks string - switch protocol { - case kubeovnv1.ProtocolIPv4: - networks = fmt.Sprintf("%s/%s", gateway, mask) - cmd = []string{MayExist, "ls-add", ls} - case kubeovnv1.ProtocolIPv6: - gateway := strings.ReplaceAll(gateway, ":", "\\:") - networks = fmt.Sprintf("%s/%s", gateway, mask) - cmd = []string{MayExist, "ls-add", ls} - case kubeovnv1.ProtocolDual: - gws := strings.Split(gateway, ",") - v6Mask := strings.Split(cidrBlocks[1], "/")[1] - gwStr := gws[0] + "/" + mask + "," + gws[1] + "/" + v6Mask - networks = strings.ReplaceAll(strings.Join(strings.Split(gwStr, ","), " "), ":", "\\:") - - cmd = []string{MayExist, "ls-add", ls} - } - if needRouter { - lsTolr := fmt.Sprintf("%s-%s", ls, lr) - lrTols := fmt.Sprintf("%s-%s", lr, ls) - - exist, err := c.LogicalSwitchPortExists(lsTolr) - if err != nil { - klog.Errorf("failed to get logical switch port %s to router, %v", lsTolr, err) - return err - } - if !exist { - cmd = append(cmd, []string{"--", MayExist, "lsp-add", ls, lsTolr, "--", - "set", "logical_switch_port", lsTolr, "type=router", "--", - "lsp-set-addresses", lsTolr, "router", "--", - "set", "logical_switch_port", lsTolr, fmt.Sprintf("options:router-port=%s", lrTols), "--", - "set", "logical_switch_port", lsTolr, fmt.Sprintf("external_ids:vendor=%s", util.CniTypeName)}...) - } - - // check router port exist - results, err := c.ListLogicalEntity("logical_router_port", fmt.Sprintf("name=%s", lrTols)) - if err != nil { - klog.Errorf("failed to list router port %s, %v", lrTols, err) - return err - } - if len(results) == 0 { - // v6address no need add \ when use lrp-add - networks = strings.ReplaceAll(networks, "\\:", ":") - networkList := strings.Split(networks, " ") - cmd = append(cmd, []string{"--", MayExist, "lrp-add", lr, lrTols, util.GenerateMac()}...) - cmd = append(cmd, networkList...) - } else { - cmd = append(cmd, []string{"--", - "set", "logical_router_port", fmt.Sprintf("%s-%s", lr, ls), fmt.Sprintf("networks=%s", networks)}...) - } - } - cmd = append(cmd, []string{"--", - "set", "logical_switch", ls, fmt.Sprintf("external_ids:vendor=%s", util.CniTypeName)}...) - _, err = c.ovnNbCommand(cmd...) - if err != nil { - klog.Errorf("set switch config for %s failed: %v", ls, err) - return err - } - return nil -} - -// CreateLogicalSwitch create logical switch in ovn, connect it to router and apply tcp/udp lb rules -func (c LegacyClient) CreateLogicalSwitch(ls, lr string, needRouter bool) error { - _, err := c.ovnNbCommand(MayExist, "ls-add", ls, "--", - "set", "logical_switch", ls, fmt.Sprintf("external_ids:vendor=%s", util.CniTypeName)) - - if err != nil { - klog.Errorf("create switch %s failed: %v", ls, err) - return err - } - - if needRouter { - if err := c.createRouterPort(ls, lr); err != nil { - klog.Errorf("failed to connect switch %s to router, %v", ls, err) - return err - } - } - return nil -} - func (c LegacyClient) AddLbToLogicalSwitch(ls string, lbs ...string) error { for _, lb := range lbs { if err := c.addLoadBalancerToLogicalSwitch(lb, ls); err != nil { @@ -522,107 +128,6 @@ func (c LegacyClient) ListLoadBalancer() ([]string, error) { return result, nil } -func (c LegacyClient) ConnectRouterToExternal(externalNet, vpcRouter, lrpIpCidr, lrpMac string, chassises []string) error { - // add lrp and lsp between vpc router and external network - lsTolr := fmt.Sprintf("%s-%s", externalNet, vpcRouter) - lrTols := fmt.Sprintf("%s-%s", vpcRouter, externalNet) - klog.Infof("add vpc lrp %s, cidr %s", lrTols, lrpIpCidr) - klog.Infof("add lsp %s", lsTolr) - _, err := c.ovnNbCommand( - MayExist, "lrp-add", vpcRouter, lrTols, lrpMac, lrpIpCidr, "--", - MayExist, "lsp-add", externalNet, lsTolr, "--", - "lsp-set-type", lsTolr, "router", "--", - "lsp-set-addresses", lsTolr, "router", "--", - "lsp-set-options", lsTolr, fmt.Sprintf("router-port=%s", lrTols), - ) - if err != nil { - return fmt.Errorf("failed to connect vpc to external, %v", err) - } - for index, chassis := range chassises { - if _, err := c.ovnNbCommand("lrp-set-gateway-chassis", lrTols, chassis, fmt.Sprintf("%d", 100-index)); err != nil { - return fmt.Errorf("failed to set gateway chassis, %v", err) - } - } - return nil -} - -func (c LegacyClient) DisconnectRouterToExternal(externalNet, vpcRouter string) error { - lrTols := fmt.Sprintf("%s-%s", vpcRouter, externalNet) - klog.Infof("delete lrp %s", lrTols) - if _, err := c.ovnNbCommand(IfExists, "lrp-del", lrTols); err != nil { - return err - } - lsTolr := fmt.Sprintf("%s-%s", externalNet, vpcRouter) - klog.Infof("delete lsp %s", lsTolr) - if _, err := c.ovnNbCommand(IfExists, "lsp-del", lsTolr); err != nil { - return err - } - return nil -} - -func (c LegacyClient) CreateGatewaySwitch(name, network string, vlan int, ip, mac string, chassises []string) error { - lsTolr := fmt.Sprintf("%s-%s", name, c.ClusterRouter) - lrTols := fmt.Sprintf("%s-%s", c.ClusterRouter, name) - klog.Infof("add vpc lrp %s, ip %s", lrTols, ip) - klog.Infof("add lsp %s", lsTolr) - localnetPort := fmt.Sprintf("ln-%s", name) - if name != util.ExternalGatewaySwitch { - // compatiable with provider network and vlan and subnet - localnetPort = GetLocalnetName(name) - } - portOptions := fmt.Sprintf("network_name=%s", network) - _, err := c.ovnNbCommand( - MayExist, "ls-add", name, "--", - "set", "logical_switch", name, fmt.Sprintf("external_ids:vendor=%s", util.CniTypeName), "--", - MayExist, "lsp-add", name, localnetPort, "--", - "lsp-set-type", localnetPort, "localnet", "--", - "lsp-set-addresses", localnetPort, "unknown", "--", - "lsp-set-options", localnetPort, portOptions, "--", - MayExist, "lrp-add", c.ClusterRouter, lrTols, mac, ip, "--", - MayExist, "lsp-add", name, lsTolr, "--", - "lsp-set-type", lsTolr, "router", "--", - "lsp-set-addresses", lsTolr, "router", "--", - "lsp-set-options", lsTolr, fmt.Sprintf("router-port=%s", lrTols), - ) - if err != nil { - return fmt.Errorf("failed to create external gateway switch, %v", err) - } - - if vlan > 0 { - portVlanId := fmt.Sprintf("tag=%d", vlan) - _, err := c.ovnNbCommand("set", "logical_switch_port", localnetPort, portVlanId) - if err != nil { - return fmt.Errorf("failed to set vlanId for ,%s, %v", localnetPort, err) - } - } - - for index, chassis := range chassises { - if _, err := c.ovnNbCommand("lrp-set-gateway-chassis", lrTols, chassis, fmt.Sprintf("%d", 100-index)); err != nil { - return fmt.Errorf("failed to set gateway chassis, %v", err) - } - } - return nil -} - -func (c LegacyClient) DeleteGatewaySwitch(name string) error { - lrTols := fmt.Sprintf("%s-%s", c.ClusterRouter, name) - klog.Infof("delete gw switch %s", name) - klog.Infof("delete gw lrp %s", lrTols) - _, err := c.ovnNbCommand( - IfExists, "ls-del", name, "--", - IfExists, "lrp-del", lrTols, - ) - return err -} - -// ListLogicalSwitch list logical switch names -func (c LegacyClient) ListLogicalSwitch(needVendorFilter bool, args ...string) ([]string, error) { - if needVendorFilter { - args = append(args, fmt.Sprintf("external_ids:vendor=%s", util.CniTypeName)) - } - return c.ListLogicalEntity("logical_switch", args...) -} - func (c LegacyClient) ListLogicalEntity(entity string, args ...string) ([]string, error) { cmd := []string{"--format=csv", "--data=bare", "--no-heading", "--columns=name", "find", entity} cmd = append(cmd, args...) @@ -701,171 +206,6 @@ func (c LegacyClient) GetEntityInfo(entity string, index string, attris []string return result, nil } -func (c LegacyClient) LogicalSwitchExists(logicalSwitch string, needVendorFilter bool, args ...string) (bool, error) { - lss, err := c.ListLogicalSwitch(needVendorFilter, args...) - if err != nil { - return false, err - } - for _, ls := range lss { - if ls == logicalSwitch { - return true, nil - } - } - return false, nil -} - -func (c LegacyClient) ListLogicalSwitchPort(needVendorFilter bool) ([]string, error) { - cmdArg := []string{"--format=csv", "--data=bare", "--no-heading", "--columns=name", "find", "logical_switch_port", "type=\"\""} - if needVendorFilter { - cmdArg = append(cmdArg, fmt.Sprintf("external_ids:vendor=%s", util.CniTypeName)) - } - output, err := c.ovnNbCommand(cmdArg...) - if err != nil { - klog.Errorf("failed to list logical switch port, %v", err) - return nil, err - } - lines := strings.Split(output, "\n") - result := make([]string, 0, len(lines)) - for _, l := range lines { - if len(strings.TrimSpace(l)) == 0 { - continue - } - result = append(result, strings.TrimSpace(l)) - } - return result, nil -} - -func (c LegacyClient) LogicalSwitchPortExists(port string) (bool, error) { - output, err := c.ovnNbCommand("--format=csv", "--data=bare", "--no-heading", "--columns=name", "find", "logical_switch_port", fmt.Sprintf("name=%s", port)) - if err != nil { - klog.Errorf("failed to find port %s: %v, %q", port, err, output) - return false, err - } - - if output != "" { - return true, nil - } - return false, nil -} - -func (c LegacyClient) ListRemoteLogicalSwitchPortAddress() ([]string, error) { - output, err := c.ovnNbCommand("--format=csv", "--data=bare", "--no-heading", "--columns=addresses", "find", "logical_switch_port", "type=remote") - if err != nil { - return nil, fmt.Errorf("failed to list ic remote addresses, %v", err) - } - lines := strings.Split(output, "\n") - result := make([]string, 0, len(lines)) - for _, l := range lines { - if len(strings.TrimSpace(l)) == 0 { - continue - } - fields := strings.Fields(l) - if len(fields) != 2 { - continue - } - cidr := fields[1] - result = append(result, strings.TrimSpace(cidr)) - } - return result, nil -} - -// ListLogicalRouter list logical router names -func (c LegacyClient) ListLogicalRouter(needVendorFilter bool, args ...string) ([]string, error) { - if needVendorFilter { - args = append(args, fmt.Sprintf("external_ids:vendor=%s", util.CniTypeName)) - } - return c.ListLogicalEntity("logical_router", args...) -} - -// DeleteLogicalSwitch delete logical switch -func (c LegacyClient) DeleteLogicalSwitch(ls string) error { - if _, err := c.ovnNbCommand(IfExists, "ls-del", ls); err != nil { - klog.Errorf("failed to del ls %s, %v", ls, err) - return err - } - return nil -} - -// CreateLogicalRouter delete logical router in ovn -func (c LegacyClient) CreateLogicalRouter(lr string) error { - _, err := c.ovnNbCommand(MayExist, "lr-add", lr, "--", - "set", "Logical_Router", lr, fmt.Sprintf("external_ids:vendor=%s", util.CniTypeName)) - return err -} - -// DeleteLogicalRouter create logical router in ovn -func (c LegacyClient) DeleteLogicalRouter(lr string) error { - _, err := c.ovnNbCommand(IfExists, "lr-del", lr) - return err -} - -func (c LegacyClient) RemoveRouterPort(ls, lr string) error { - lsTolr := fmt.Sprintf("%s-%s", ls, lr) - lrTols := fmt.Sprintf("%s-%s", lr, ls) - klog.Infof("remove router port %s, switch port %s", lrTols, lsTolr) - _, err := c.ovnNbCommand(IfExists, "lsp-del", lsTolr, "--", - IfExists, "lrp-del", lrTols) - if err != nil { - klog.Errorf("failed to remove router port, %v", err) - return err - } - return nil -} - -func (c LegacyClient) createRouterPort(ls, lr string) error { - klog.Infof("add %s to %s", ls, lr) - lsTolr := fmt.Sprintf("%s-%s", ls, lr) - lrTols := fmt.Sprintf("%s-%s", lr, ls) - _, err := c.ovnNbCommand(MayExist, "lsp-add", ls, lsTolr, "--", - "set", "logical_switch_port", lsTolr, "type=router", "--", - "lsp-set-addresses", lsTolr, "router", "--", - "set", "logical_switch_port", lsTolr, fmt.Sprintf("options:router-port=%s", lrTols), "--", - "set", "logical_switch_port", lsTolr, fmt.Sprintf("external_ids:vendor=%s", util.CniTypeName)) - if err != nil { - klog.Errorf("failed to create switch router port %s: %v", lsTolr, err) - return err - } - return nil -} - -func (c LegacyClient) CreatePeerRouterPort(localRouter, remoteRouter, localRouterPortIP string) error { - localRouterPort := fmt.Sprintf("%s-%s", localRouter, remoteRouter) - remoteRouterPort := fmt.Sprintf("%s-%s", remoteRouter, localRouter) - - // check router port exist, because '--may-exist' may not work for router port - results, err := c.ListLogicalEntity("logical_router_port", fmt.Sprintf("name=%s", localRouterPort)) - if err != nil { - klog.Errorf("failed to list router port %s, %v", localRouterPort, err) - return err - } - if len(results) == 0 { - ipStr := strings.Split(localRouterPortIP, ",") - if len(ipStr) == 2 { - klog.Infof("add vpc lrp %s", localRouterPort) - _, err = c.ovnNbCommand(MayExist, "lrp-add", localRouter, localRouterPort, util.GenerateMac(), ipStr[0], ipStr[1], "--", - "set", "logical_router_port", localRouterPort, fmt.Sprintf("peer=%s", remoteRouterPort)) - } else { - klog.Infof("add vpc lrp %s", localRouterPort) - _, err = c.ovnNbCommand(MayExist, "lrp-add", localRouter, localRouterPort, util.GenerateMac(), ipStr[0], "--", - "set", "logical_router_port", localRouterPort, fmt.Sprintf("peer=%s", remoteRouterPort)) - } - if err != nil { - klog.Errorf("failed to create router port %s: %v", localRouterPort, err) - return err - } - } - - klog.Infof("set lrp %s, networks %s", localRouterPort, localRouterPortIP) - _, err = c.ovnNbCommand("set", "logical_router_port", localRouterPort, - fmt.Sprintf("networks=\"%s\"", strings.ReplaceAll(localRouterPortIP, ",", " "))) - - if err != nil { - klog.Errorf("failed to set router port %s: %v", localRouterPort, err) - return err - } - return nil -} - type StaticRoute struct { Policy string CIDR string @@ -1556,65 +896,6 @@ func (c LegacyClient) SetPrivateLogicalSwitch(ls, cidr string, allow []string) e return err } -func (c LegacyClient) GetLogicalSwitchPortAddress(port string) ([]string, error) { - output, err := c.ovnNbCommand("get", "logical_switch_port", port, "addresses") - if err != nil { - klog.Errorf("get port %s addresses failed: %v", port, err) - return nil, err - } - if strings.Contains(output, "dynamic") { - // [dynamic] - return nil, nil - } - output = strings.Trim(output, `[]"`) - fields := strings.Fields(output) - if len(fields) != 2 { - return nil, nil - } - - // currently user may only have one fixed address - // ["0a:00:00:00:00:0c 10.16.0.13"] - return fields, nil -} - -func (c LegacyClient) GetLogicalSwitchPortDynamicAddress(port string) ([]string, error) { - output, err := c.ovnNbCommand("wait-until", "logical_switch_port", port, "dynamic_addresses!=[]", "--", - "get", "logical_switch_port", port, "dynamic-addresses") - if err != nil { - klog.Errorf("get port %s dynamic_addresses failed: %v", port, err) - return nil, err - } - if output == "[]" { - return nil, ErrNoAddr - } - output = strings.Trim(output, `"`) - // "0a:00:00:00:00:02" - fields := strings.Fields(output) - if len(fields) != 2 { - klog.Error("Subnet address space has been exhausted") - return nil, ErrNoAddr - } - // "0a:00:00:00:00:02 100.64.0.3" - return fields, nil -} - -// GetPortAddr return port [mac, ip] -func (c LegacyClient) GetPortAddr(port string) ([]string, error) { - var address []string - var err error - address, err = c.GetLogicalSwitchPortAddress(port) - if err != nil { - return nil, err - } - if address == nil { - address, err = c.GetLogicalSwitchPortDynamicAddress(port) - if err != nil { - return nil, err - } - } - return address, nil -} - func (c LegacyClient) CreateNpPortGroup(pgName, npNs, npName string) error { output, err := c.ovnNbCommand( "--data=bare", "--no-heading", "--columns=_uuid", "find", "port_group", fmt.Sprintf("name=%s", pgName)) @@ -2157,63 +1438,6 @@ func CheckAlive() error { return nil } -// GetLogicalSwitchExcludeIPS get a logical switch exclude ips -// ovn-nbctl get logical_switch ovn-default other_config:exclude_ips => "10.17.0.1 10.17.0.2 10.17.0.3..10.17.0.5" -func (c LegacyClient) GetLogicalSwitchExcludeIPS(logicalSwitch string) ([]string, error) { - output, err := c.ovnNbCommand(IfExists, "get", "logical_switch", logicalSwitch, "other_config:exclude_ips") - if err != nil { - return nil, err - } - output = strings.Trim(output, `"`) - if output == "" { - return nil, ErrNoAddr - } - return strings.Fields(output), nil -} - -// SetLogicalSwitchExcludeIPS set a logical switch exclude ips -// ovn-nbctl set logical_switch ovn-default other_config:exclude_ips="10.17.0.2 10.17.0.1" -func (c LegacyClient) SetLogicalSwitchExcludeIPS(logicalSwitch string, excludeIPS []string) error { - _, err := c.ovnNbCommand("set", "logical_switch", logicalSwitch, - fmt.Sprintf(`other_config:exclude_ips="%s"`, strings.Join(excludeIPS, " "))) - return err -} - -func (c LegacyClient) GetLogicalSwitchPortByLogicalSwitch(logicalSwitch string) ([]string, error) { - output, err := c.ovnNbCommand("lsp-list", logicalSwitch) - if err != nil { - return nil, err - } - var rv []string - lines := strings.Split(output, "\n") - for _, line := range lines { - lsp := strings.Fields(line)[0] - rv = append(rv, lsp) - } - return rv, nil -} - -func (c LegacyClient) CreateLocalnetPort(ls, port, provider string, vlanID int) error { - cmdArg := []string{ - MayExist, "lsp-add", ls, port, "--", - "lsp-set-addresses", port, "unknown", "--", - "lsp-set-type", port, "localnet", "--", - "lsp-set-options", port, fmt.Sprintf("network_name=%s", provider), "--", - "set", "logical_switch_port", port, fmt.Sprintf("external_ids:vendor=%s", util.CniTypeName), - } - if vlanID > 0 && vlanID < 4096 { - cmdArg = append(cmdArg, - "--", "set", "logical_switch_port", port, fmt.Sprintf("tag=%d", vlanID)) - } - - if _, err := c.ovnNbCommand(cmdArg...); err != nil { - klog.Errorf("create localnet port %s failed, %v", port, err) - return err - } - - return nil -} - func GetSgPortGroupName(sgName string) string { return strings.Replace(fmt.Sprintf("ovn.sg.%s", sgName), "-", ".", -1) } @@ -2505,33 +1729,6 @@ func (c LegacyClient) UpdateSgACL(sg *kubeovnv1.SecurityGroup, direction AclDire } return nil } -func (c LegacyClient) OvnGet(table, record, column, key string) (string, error) { - var columnVal string - if key == "" { - columnVal = column - } else { - columnVal = column + ":" + key - } - args := []string{"get", table, record, columnVal} - return c.ovnNbCommand(args...) -} - -func (c LegacyClient) SetLspExternalIds(name string, externalIDs map[string]string) error { - if len(externalIDs) == 0 { - return nil - } - - cmd := make([]string, len(externalIDs)+3) - cmd = append(cmd, "set", "logical_switch_port", name) - for k, v := range externalIDs { - cmd = append(cmd, fmt.Sprintf(`external-ids:%s="%s"`, k, v)) - } - - if _, err := c.ovnNbCommand(cmd...); err != nil { - return fmt.Errorf("failed to set external-ids for logical switch port %s: %v", name, err) - } - return nil -} func (c *LegacyClient) AclExists(priority, direction string) (bool, error) { priorityVal, _ := strconv.Atoi(priority) @@ -2546,13 +1743,6 @@ func (c *LegacyClient) AclExists(priority, direction string) (bool, error) { return true, nil } -func (c *LegacyClient) SetLBCIDR(svccidr string) error { - if _, err := c.ovnNbCommand("set", "NB_Global", ".", fmt.Sprintf("options:svc_ipv4_cidr=%s", svccidr)); err != nil { - return fmt.Errorf("failed to set svc cidr for lb, %v", err) - } - return nil -} - func (c *LegacyClient) PortGroupExists(pgName string) (bool, error) { results, err := c.CustomFindEntity("port_group", []string{"_uuid"}, fmt.Sprintf("name=%s", pgName)) if err != nil { @@ -2940,48 +2130,6 @@ func (c *LegacyClient) DeleteDHCPOptions(ls string, protocol string) error { return c.DeleteDHCPOptionsByUUIDs(uuidToDeleteList) } -func (c *LegacyClient) UpdateRouterPortIPv6RA(ls, lr, cidrBlock, gateway, ipv6RAConfigsStr string, enableIPv6RA bool) error { - var err error - lrTols := fmt.Sprintf("%s-%s", lr, ls) - ip := util.GetIpAddrWithMask(gateway, cidrBlock) - ipStr := strings.Split(ip, ",") - if enableIPv6RA { - var ipv6Prefix string - switch util.CheckProtocol(ip) { - case kubeovnv1.ProtocolIPv4: - klog.Warningf("enable ipv6 router advertisement is not effective to IPv4") - return nil - case kubeovnv1.ProtocolIPv6: - ipv6Prefix = strings.Split(ipStr[0], "/")[1] - case kubeovnv1.ProtocolDual: - ipv6Prefix = strings.Split(ipStr[1], "/")[1] - } - - if len(ipv6RAConfigsStr) == 0 { - // default ipv6_ra_configs - ipv6RAConfigsStr = "address_mode=dhcpv6_stateful,max_interval=30,min_interval=5,send_periodic=true" - } - - ipv6RAConfigsStr = strings.ReplaceAll(ipv6RAConfigsStr, " ", "") - klog.Infof("set lrp %s, ipv6_prefix %s", lrTols, ipv6Prefix) - _, err = c.ovnNbCommand("--", - "set", "logical_router_port", lrTols, fmt.Sprintf("ipv6_prefix=%s", ipv6Prefix), fmt.Sprintf("ipv6_ra_configs=%s", ipv6RAConfigsStr)) - if err != nil { - klog.Errorf("failed to set ipv6_prefix: %s ans ipv6_ra_configs: %s for router port: %s, err: %s", ipv6Prefix, ipv6RAConfigsStr, lrTols, err) - return err - } - } else { - klog.Infof("set lrp %s", lrTols) - _, err = c.ovnNbCommand("--", - "set", "logical_router_port", lrTols, "ipv6_prefix=[]", "ipv6_ra_configs={}") - if err != nil { - klog.Errorf("failed to reset ipv6_prefix and ipv6_ra_config for router port: %s, err: %s", lrTols, err) - return err - } - } - return nil -} - func (c LegacyClient) DeleteSubnetACL(ls string) error { results, err := c.CustomFindEntity("acl", []string{"direction", "priority", "match"}, fmt.Sprintf("external_ids:subnet=\"%s\"", ls)) if err != nil { @@ -3108,20 +2256,6 @@ func (c LegacyClient) SetAclLog(pgName string, logEnable, isIngress bool) error return nil } -func (c *LegacyClient) GetRouterNat(routerName string) ([]string, error) { - var nat []string - results, err := c.CustomFindEntity("logical-router", []string{"nat"}, fmt.Sprintf("name=%s", routerName)) - if err != nil { - klog.Errorf("customFindEntity failed, %v", err) - return nat, err - } - if len(results) == 0 { - return nat, nil - } - - return results[0]["nat"], nil -} - func (c *LegacyClient) GetNatIPInfo(uuid string) (string, error) { var logical_ip string diff --git a/pkg/ovs/ovn.go b/pkg/ovs/ovn.go index f878120fc25..ba64a680546 100644 --- a/pkg/ovs/ovn.go +++ b/pkg/ovs/ovn.go @@ -114,7 +114,7 @@ func ConstructWaitForUniqueOperation(table string, column string, value interfac func (c *ovnClient) Transact(method string, operations []ovsdb.Operation) error { if len(operations) == 0 { - klog.Warningf("operations should not be empty") + klog.V(6).Info("operations should not be empty") return nil } diff --git a/pkg/util/net.go b/pkg/util/net.go index 8de383d493c..9a95cf93b0a 100644 --- a/pkg/util/net.go +++ b/pkg/util/net.go @@ -108,14 +108,24 @@ func LastIP(subnet string) (string, error) { } func CIDRContainIP(cidrStr, ipStr string) bool { - var containFlag bool - for _, cidr := range strings.Split(cidrStr, ",") { + cidrs := strings.Split(cidrStr, ",") + ips := strings.Split(ipStr, ",") + + if len(cidrs) == 1 { + for _, ip := range ips { + if CheckProtocol(cidrStr) != CheckProtocol(ip) { + return false + } + } + } + + for _, cidr := range cidrs { _, cidrNet, err := net.ParseCIDR(cidr) if err != nil { return false } - for _, ip := range strings.Split(ipStr, ",") { + for _, ip := range ips { if CheckProtocol(cidr) != CheckProtocol(ip) { continue } @@ -124,15 +134,13 @@ func CIDRContainIP(cidrStr, ipStr string) bool { return false } - if cidrNet.Contains(ipAddr) { - containFlag = true - } else { - containFlag = false + if !cidrNet.Contains(ipAddr) { + return false } } } - // v4 and v6 address should be both matched for dual-stack check - return containFlag + // v4 and v6 address should be both matched for dualstack check + return true } func CheckProtocol(address string) string { diff --git a/pkg/util/net_test.go b/pkg/util/net_test.go index 116551c8be5..456c857215c 100644 --- a/pkg/util/net_test.go +++ b/pkg/util/net_test.go @@ -7,6 +7,7 @@ import ( "testing" kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1" + "github.com/stretchr/testify/require" ) func TestCheckSystemCIDR(t *testing.T) { @@ -994,3 +995,76 @@ func TestJoinHostPort(t *testing.T) { }) } } + +func Test_CIDRContainIP(t *testing.T) { + t.Parallel() + + tests := []struct { + desc string + cidrs string + ips string + want bool + }{ + { + "ipv4 family", + "192.168.230.0/24", + "192.168.230.10,192.168.230.11", + true, + }, + { + "ipv4 family which CIDR does't contain ip", + "192.168.230.0/24", + "192.168.231.10,192.168.230.11", + false, + }, + { + "ipv6 family", + "fc00::0af4:00/112", + "fc00::0af4:10,fc00::0af4:11", + true, + }, + { + "ipv6 family which CIDR does't contain ip", + "fd00::c0a8:d200/120", + "fc00::c0a8:d210", + false, + }, + { + "dual", + "192.168.230.0/24,fc00::0af4:00/112", + "fc00::0af4:10,fc00::0af4:11,192.168.230.10,192.168.230.11", + true, + }, + { + "dual which CIDR does't contain ip", + "192.168.230.0/24,fc00::0af4:00/112", + "fc00::0af4:10,fd00::0af4:11,192.168.230.10,192.168.230.11", + false, + }, + { + "different family", + "fd00::c0a8:d200/120", + "10.96.0.1", + false, + }, + { + "different family", + "10.96.0.0/16", + "fd00::c0a8:d201", + false, + }, + { + "ipv4 family which CIDR has no mask", + "192.168.0.23", + "192.168.0.23,192.168.0.254", + false, + }, + } + + for _, tt := range tests { + t.Run(tt.desc, func(t *testing.T) { + got := CIDRContainIP(tt.cidrs, tt.ips) + require.Equal(t, got, tt.want) + }) + } +} diff --git a/pkg/util/slice_test.go b/pkg/util/slice_test.go index 8f073919b20..387ac769c91 100644 --- a/pkg/util/slice_test.go +++ b/pkg/util/slice_test.go @@ -3,6 +3,8 @@ package util import ( "reflect" "testing" + + "github.com/stretchr/testify/require" ) func TestDiffStringSlice(t *testing.T) { @@ -182,3 +184,50 @@ func TestRemoveString(t *testing.T) { }) } } +func Test_DiffStringSlice(t *testing.T) { + t.Parallel() + tests := []struct { + desc string + s1 []string + s2 []string + diff []string + }{ + { + "slice1 is subset of slice2", + []string{"1", "2", "3"}, + []string{"1", "2", "3", "4", "5"}, + []string{"4", "5"}, + }, + { + "slice2 is subset of slice1", + []string{"1", "2", "3", "4", "5"}, + []string{"1", "2", "3"}, + []string{"4", "5"}, + }, + { + "slice1 is empty", + nil, + []string{"1", "2", "3", "4", "5"}, + []string{"1", "2", "3", "4", "5"}, + }, + { + "slice2 is empty", + []string{"1", "2", "3", "4", "5"}, + nil, + []string{"1", "2", "3", "4", "5"}, + }, + { + "slice1 and slice2 have intersection", + []string{"1", "2", "3"}, + []string{"6", "7", "3", "1", "5"}, + []string{"2", "5", "6", "7"}, + }, + } + + for _, tt := range tests { + t.Run(tt.desc, func(t *testing.T) { + diff := DiffStringSlice(tt.s1, tt.s2) + require.ElementsMatch(t, tt.diff, diff) + }) + } +} diff --git a/pkg/util/validator_test.go b/pkg/util/validator_test.go index fd12a8fdb3e..b395c86303c 100644 --- a/pkg/util/validator_test.go +++ b/pkg/util/validator_test.go @@ -561,7 +561,7 @@ func TestValidatePodNetwork(t *testing.T) { "ovn.kubernetes.io/egress_rate": "1", "ovn.kubernetes.io/cidr": "10.16.0.0/16", }, - err: "[10.244.0.0/16 not in cidr 10.16.0.0/16, fd00:10:244:0:2::/80 not in cidr 10.16.0.0/16]", + err: "10.244.0.0/16 not in cidr 10.16.0.0/16", }, { name: "podMacErr", @@ -585,7 +585,7 @@ func TestValidatePodNetwork(t *testing.T) { "ovn.kubernetes.io/egress_rate": "1", "ovn.kubernetes.io/cidr": "10.16.0.0/16", }, - err: "10.16.1111.15 in ovn.kubernetes.io/ip_pool is not a valid address", + err: "10.16.1111.15,10.16.0.16,10.16.0.17 not in cidr 10.16.0.0/16", }, { name: "ingRaErr",