Skip to content

Commit

Permalink
do not delete external switch if it is created by provider network vl…
Browse files Browse the repository at this point in the history 
…an subnet (#2449)

* rename ecmp to ecmp mode

* do not delete external switch if it is created by provider network vlan subnet

* delete lrp eip should delete lrp
  • Loading branch information
@bobz965
bobz965 committed Mar 13, 2023
1 parent 282706d commit 5387acf
Show file tree
Hide file tree
Showing 14 changed files with 215 additions and 141 deletions.
2 changes: 1 addition & 1 deletion dist/images/install.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1040,7 +1040,7 @@ spec:
type: string
nextHopIP:
type: string
ecmp:
ecmpMode:
type: string
bfdId:
type: string
Expand Down
2 changes: 1 addition & 1 deletion kubeovn-helm/templates/kube-ovn-crd.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -822,7 +822,7 @@ spec:
type: string
nextHopIP:
type: string
ecmp:
ecmpMode:
type: string
bfdId:
type: string
Expand Down
2 changes: 1 addition & 1 deletion pkg/apis/kubeovn/v1/types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -375,7 +375,7 @@ type StaticRoute struct {
Policy RoutePolicy `json:"policy,omitempty"`
CIDR string `json:"cidr"`
NextHopIP string `json:"nextHopIP"`
ECMP string `json:"ecmp"`
ECMPMode string `json:"ecmpMode"`
BfdId string `json:"bfdId"`
}

Expand Down
131 changes: 61 additions & 70 deletions pkg/controller/controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -578,50 +578,48 @@ func NewController(config *Configuration) *Controller {
}); err != nil {
util.LogFatalAndExit(err, "failed to add iptables snat rule event handler")
}
if config.EnableEipSnat {
ovnEipInformer := kubeovnInformerFactory.Kubeovn().V1().OvnEips()
controller.ovnEipsLister = ovnEipInformer.Lister()
controller.ovnEipSynced = ovnEipInformer.Informer().HasSynced
controller.addOvnEipQueue = workqueue.NewNamedRateLimitingQueue(custCrdRateLimiter, "addOvnEip")
controller.updateOvnEipQueue = workqueue.NewNamedRateLimitingQueue(custCrdRateLimiter, "updateOvnEip")
controller.resetOvnEipQueue = workqueue.NewNamedRateLimitingQueue(custCrdRateLimiter, "resetOvnEip")
controller.delOvnEipQueue = workqueue.NewNamedRateLimitingQueue(custCrdRateLimiter, "delOvnEip")

if _, err = ovnEipInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
AddFunc: controller.enqueueAddOvnEip,
UpdateFunc: controller.enqueueUpdateOvnEip,
DeleteFunc: controller.enqueueDelOvnEip,
}); err != nil {
util.LogFatalAndExit(err, "failed to add eip event handler")
}
ovnEipInformer := kubeovnInformerFactory.Kubeovn().V1().OvnEips()
controller.ovnEipsLister = ovnEipInformer.Lister()
controller.ovnEipSynced = ovnEipInformer.Informer().HasSynced
controller.addOvnEipQueue = workqueue.NewNamedRateLimitingQueue(custCrdRateLimiter, "addOvnEip")
controller.updateOvnEipQueue = workqueue.NewNamedRateLimitingQueue(custCrdRateLimiter, "updateOvnEip")
controller.resetOvnEipQueue = workqueue.NewNamedRateLimitingQueue(custCrdRateLimiter, "resetOvnEip")
controller.delOvnEipQueue = workqueue.NewNamedRateLimitingQueue(custCrdRateLimiter, "delOvnEip")

if _, err = ovnEipInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
AddFunc: controller.enqueueAddOvnEip,
UpdateFunc: controller.enqueueUpdateOvnEip,
DeleteFunc: controller.enqueueDelOvnEip,
}); err != nil {
util.LogFatalAndExit(err, "failed to add eip event handler")
}

ovnFipInformer := kubeovnInformerFactory.Kubeovn().V1().OvnFips()
controller.ovnFipsLister = ovnFipInformer.Lister()
controller.ovnFipSynced = ovnFipInformer.Informer().HasSynced
controller.addOvnFipQueue = workqueue.NewNamedRateLimitingQueue(custCrdRateLimiter, "addOvnFip")
controller.updateOvnFipQueue = workqueue.NewNamedRateLimitingQueue(custCrdRateLimiter, "updateOvnFip")
controller.delOvnFipQueue = workqueue.NewNamedRateLimitingQueue(custCrdRateLimiter, "delOvnFip")
if _, err = ovnFipInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
AddFunc: controller.enqueueAddOvnFip,
UpdateFunc: controller.enqueueUpdateOvnFip,
DeleteFunc: controller.enqueueDelOvnFip,
}); err != nil {
util.LogFatalAndExit(err, "failed to add ovn fip event handler")
}
ovnFipInformer := kubeovnInformerFactory.Kubeovn().V1().OvnFips()
controller.ovnFipsLister = ovnFipInformer.Lister()
controller.ovnFipSynced = ovnFipInformer.Informer().HasSynced
controller.addOvnFipQueue = workqueue.NewNamedRateLimitingQueue(custCrdRateLimiter, "addOvnFip")
controller.updateOvnFipQueue = workqueue.NewNamedRateLimitingQueue(custCrdRateLimiter, "updateOvnFip")
controller.delOvnFipQueue = workqueue.NewNamedRateLimitingQueue(custCrdRateLimiter, "delOvnFip")
if _, err = ovnFipInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
AddFunc: controller.enqueueAddOvnFip,
UpdateFunc: controller.enqueueUpdateOvnFip,
DeleteFunc: controller.enqueueDelOvnFip,
}); err != nil {
util.LogFatalAndExit(err, "failed to add ovn fip event handler")
}

ovnSnatRuleInformer := kubeovnInformerFactory.Kubeovn().V1().OvnSnatRules()
controller.ovnSnatRulesLister = ovnSnatRuleInformer.Lister()
controller.ovnSnatRuleSynced = ovnSnatRuleInformer.Informer().HasSynced
controller.addOvnSnatRuleQueue = workqueue.NewNamedRateLimitingQueue(custCrdRateLimiter, "addOvnSnatRule")
controller.updateOvnSnatRuleQueue = workqueue.NewNamedRateLimitingQueue(custCrdRateLimiter, "updateOvnSnatRule")
controller.delOvnSnatRuleQueue = workqueue.NewNamedRateLimitingQueue(custCrdRateLimiter, "delOvnSnatRule")
if _, err = ovnSnatRuleInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
AddFunc: controller.enqueueAddOvnSnatRule,
UpdateFunc: controller.enqueueUpdateOvnSnatRule,
DeleteFunc: controller.enqueueDelOvnSnatRule,
}); err != nil {
util.LogFatalAndExit(err, "failed to add ovn snat rule event handler")
}
ovnSnatRuleInformer := kubeovnInformerFactory.Kubeovn().V1().OvnSnatRules()
controller.ovnSnatRulesLister = ovnSnatRuleInformer.Lister()
controller.ovnSnatRuleSynced = ovnSnatRuleInformer.Informer().HasSynced
controller.addOvnSnatRuleQueue = workqueue.NewNamedRateLimitingQueue(custCrdRateLimiter, "addOvnSnatRule")
controller.updateOvnSnatRuleQueue = workqueue.NewNamedRateLimitingQueue(custCrdRateLimiter, "updateOvnSnatRule")
controller.delOvnSnatRuleQueue = workqueue.NewNamedRateLimitingQueue(custCrdRateLimiter, "delOvnSnatRule")
if _, err = ovnSnatRuleInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
AddFunc: controller.enqueueAddOvnSnatRule,
UpdateFunc: controller.enqueueUpdateOvnSnatRule,
DeleteFunc: controller.enqueueDelOvnSnatRule,
}); err != nil {
util.LogFatalAndExit(err, "failed to add ovn snat rule event handler")
}

if _, err = podAnnotatedIptablesEipInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
Expand Down Expand Up @@ -662,10 +660,7 @@ func (c *Controller) Run(ctx context.Context) {
c.podAnnotatedIptablesEipSynced, c.podAnnotatedIptablesFipSynced,
c.vlanSynced, c.podsSynced, c.namespacesSynced, c.nodesSynced,
c.serviceSynced, c.endpointsSynced, c.configMapsSynced,
}

if c.config.EnableEipSnat {
cacheSyncs = append(cacheSyncs, c.ovnEipSynced, c.ovnFipSynced, c.ovnSnatRuleSynced)
c.ovnEipSynced, c.ovnFipSynced, c.ovnSnatRuleSynced,
}

if c.config.EnableNP {
Expand Down Expand Up @@ -824,20 +819,18 @@ func (c *Controller) shutdown() {
c.updateIptablesSnatRuleQueue.ShutDown()
c.delIptablesSnatRuleQueue.ShutDown()

if c.config.EnableEipSnat {
c.addOvnEipQueue.ShutDown()
c.updateOvnEipQueue.ShutDown()
c.resetOvnEipQueue.ShutDown()
c.delOvnEipQueue.ShutDown()
c.addOvnEipQueue.ShutDown()
c.updateOvnEipQueue.ShutDown()
c.resetOvnEipQueue.ShutDown()
c.delOvnEipQueue.ShutDown()

c.addOvnFipQueue.ShutDown()
c.updateOvnFipQueue.ShutDown()
c.delOvnFipQueue.ShutDown()
c.addOvnFipQueue.ShutDown()
c.updateOvnFipQueue.ShutDown()
c.delOvnFipQueue.ShutDown()

c.addIptablesSnatRuleQueue.ShutDown()
c.updateIptablesSnatRuleQueue.ShutDown()
c.delIptablesSnatRuleQueue.ShutDown()
}
c.addOvnSnatRuleQueue.ShutDown()
c.updateOvnSnatRuleQueue.ShutDown()
c.delOvnSnatRuleQueue.ShutDown()

if c.config.PodDefaultFipType == util.IptablesFip {
c.addPodAnnotatedIptablesEipQueue.ShutDown()
Expand Down Expand Up @@ -999,20 +992,18 @@ func (c *Controller) startWorkers(ctx context.Context) {
go wait.Until(c.resyncSubnetMetrics, 30*time.Second, ctx.Done())
go wait.Until(c.CheckGatewayReady, 5*time.Second, ctx.Done())

if c.config.EnableEipSnat {
go wait.Until(c.runAddOvnEipWorker, time.Second, ctx.Done())
go wait.Until(c.runUpdateOvnEipWorker, time.Second, ctx.Done())
go wait.Until(c.runResetOvnEipWorker, time.Second, ctx.Done())
go wait.Until(c.runDelOvnEipWorker, time.Second, ctx.Done())
go wait.Until(c.runAddOvnEipWorker, time.Second, ctx.Done())
go wait.Until(c.runUpdateOvnEipWorker, time.Second, ctx.Done())
go wait.Until(c.runResetOvnEipWorker, time.Second, ctx.Done())
go wait.Until(c.runDelOvnEipWorker, time.Second, ctx.Done())

go wait.Until(c.runAddOvnFipWorker, time.Second, ctx.Done())
go wait.Until(c.runUpdateOvnFipWorker, time.Second, ctx.Done())
go wait.Until(c.runDelOvnFipWorker, time.Second, ctx.Done())
go wait.Until(c.runAddOvnFipWorker, time.Second, ctx.Done())
go wait.Until(c.runUpdateOvnFipWorker, time.Second, ctx.Done())
go wait.Until(c.runDelOvnFipWorker, time.Second, ctx.Done())

go wait.Until(c.runAddOvnSnatRuleWorker, time.Second, ctx.Done())
go wait.Until(c.runUpdateOvnSnatRuleWorker, time.Second, ctx.Done())
go wait.Until(c.runDelOvnSnatRuleWorker, time.Second, ctx.Done())
}
go wait.Until(c.runAddOvnSnatRuleWorker, time.Second, ctx.Done())
go wait.Until(c.runUpdateOvnSnatRuleWorker, time.Second, ctx.Done())
go wait.Until(c.runDelOvnSnatRuleWorker, time.Second, ctx.Done())

if c.config.EnableNP {
go wait.Until(c.CheckNodePortGroup, time.Duration(c.config.NodePgProbeTime)*time.Minute, ctx.Done())
Expand Down
107 changes: 70 additions & 37 deletions pkg/controller/external-gw.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,10 @@ func (c *Controller) resyncExternalGateway() {
klog.Errorf("failed to remove ovn external gw, %v", err)
return
}
if err := c.updateDefaultVpcExternal(false); err != nil {
klog.Error("failed to update default vpc, %v", err)
return
}
exGwEnabled = "false"
lastExGwCM = nil
klog.Info("finish remove ovn external gw")
Expand All @@ -63,41 +67,11 @@ func (c *Controller) resyncExternalGateway() {
lastExGwCM = cm.Data
c.ovnLegacyClient.ExternalGatewayType = cm.Data["type"]
c.ExternalGatewayType = cm.Data["type"]
klog.Info("finish establishing ovn external gw")

cachedVpc, err := c.vpcsLister.Get(c.config.ClusterRouter)
if err != nil {
klog.Errorf("failed to get vpc %s, %v", c.config.ClusterRouter, err)
return
}
vpc := cachedVpc.DeepCopy()
if !vpc.Spec.EnableExternal {
vpc.Spec.EnableExternal = true
if _, err := c.config.KubeOvnClient.KubeovnV1().Vpcs().Update(context.Background(), vpc, metav1.UpdateOptions{}); err != nil {
errMsg := fmt.Errorf("failed to update vpc enable external %s, %v", vpc.Name, err)
klog.Error(errMsg)
return
}
}
if !vpc.Status.EnableExternal {
vpc.Status.EnableExternal = true
bytes, err := vpc.Status.Bytes()
if err != nil {
klog.Errorf("failed to get vpc bytes, %v", err)
return
}
if _, err = c.config.KubeOvnClient.KubeovnV1().Vpcs().Patch(context.Background(),
vpc.Name, types.MergePatchType, bytes, metav1.PatchOptions{}, "status"); err != nil {
klog.Errorf("failed to patch vpc %s, %v", c.config.ClusterRouter, err)
return
}
}

lrpEipName := fmt.Sprintf("%s-%s", util.DefaultVpc, c.config.ExternalGatewaySwitch)
if err := c.patchLrpOvnEipEnableBfdLabel(lrpEipName, vpc.Spec.EnableBfd); err != nil {
klog.Errorf("failed to patch label for lrp %s, %v", lrpEipName, err)
if err := c.updateDefaultVpcExternal(true); err != nil {
klog.Error("failed to update default vpc, %v", err)
return
}
klog.Info("finish establishing ovn external gw")
}
}

Expand Down Expand Up @@ -130,10 +104,33 @@ func (c *Controller) removeExternalGateway() error {
}
}

klog.Infof("delete external gateway switch %s", c.config.ExternalGatewaySwitch)
if err := c.ovnLegacyClient.DeleteGatewaySwitch(c.config.ExternalGatewaySwitch); err != nil {
klog.Errorf("failed to delete external gateway switch, %v", err)
return err
keepExternalSubnet := false
externalSubnet, err := c.subnetsLister.Get(c.config.ExternalGatewaySwitch)
if err != nil {
if !k8serrors.IsNotFound(err) {
klog.Errorf("failed to get subnet %s, %v", c.config.ExternalGatewaySwitch, err)
return err
}
} else {
if externalSubnet.Spec.Vlan != "" {
keepExternalSubnet = true
}
}

if !keepExternalSubnet {
klog.Infof("delete external gateway switch %s", c.config.ExternalGatewaySwitch)
if err := c.ovnLegacyClient.DeleteGatewaySwitch(c.config.ExternalGatewaySwitch); err != nil {
klog.Errorf("failed to delete external gateway switch, %v", err)
return err
}
} else {
klog.Infof("should keep provider network vlan underlay external gateway switch %s", c.config.ExternalGatewaySwitch)
lrpName := fmt.Sprintf("%s-%s", c.config.ClusterRouter, c.config.ExternalGatewaySwitch)
klog.Infof("delete logical router port %s", lrpName)
if err := c.ovnLegacyClient.DeleteLogicalRouterPort(lrpName); err != nil {
klog.Errorf("failed to delete lrp %s, %v", lrpName, err)
return err
}
}
return nil
}
Expand Down Expand Up @@ -267,3 +264,39 @@ func (c *Controller) getGatewayChassis(config map[string]string) ([]string, erro
}
return chassises, nil
}

func (c *Controller) updateDefaultVpcExternal(enableExternal bool) error {
cachedVpc, err := c.vpcsLister.Get(c.config.ClusterRouter)
if err != nil {
klog.Errorf("failed to get vpc %s, %v", c.config.ClusterRouter, err)
return err
}
vpc := cachedVpc.DeepCopy()
if vpc.Spec.EnableExternal != enableExternal {
vpc.Spec.EnableExternal = enableExternal
if _, err := c.config.KubeOvnClient.KubeovnV1().Vpcs().Update(context.Background(), vpc, metav1.UpdateOptions{}); err != nil {
errMsg := fmt.Errorf("failed to update vpc enable external %s, %v", vpc.Name, err)
klog.Error(errMsg)
return err
}
}
if vpc.Status.EnableExternal != enableExternal {
vpc.Status.EnableExternal = enableExternal
bytes, err := vpc.Status.Bytes()
if err != nil {
klog.Errorf("failed to get vpc bytes, %v", err)
return err
}
if _, err = c.config.KubeOvnClient.KubeovnV1().Vpcs().Patch(context.Background(),
vpc.Name, types.MergePatchType, bytes, metav1.PatchOptions{}, "status"); err != nil {
klog.Errorf("failed to patch vpc %s, %v", c.config.ClusterRouter, err)
return err
}
lrpEipName := fmt.Sprintf("%s-%s", util.DefaultVpc, c.config.ExternalGatewaySwitch)
if err := c.patchLrpOvnEipEnableBfdLabel(lrpEipName, vpc.Spec.EnableBfd); err != nil {
klog.Errorf("failed to patch label for lrp %s, %v", lrpEipName, err)
return err
}
}
return nil
}
18 changes: 8 additions & 10 deletions pkg/controller/init.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -422,16 +422,14 @@ func (c *Controller) InitIPAM() error {
klog.Errorf("failed to init ipam from iptables eip cr %s: %v", eip.Name, err)
}
}
if c.config.EnableEipSnat {
oeips, err := c.ovnEipsLister.List(labels.Everything())
if err != nil {
klog.Errorf("failed to list ovn eips: %v", err)
return err
}
for _, oeip := range oeips {
if _, _, _, err = c.ipam.GetStaticAddress(oeip.Name, oeip.Name, oeip.Status.V4Ip, oeip.Status.MacAddress, oeip.Spec.ExternalSubnet, true); err != nil {
klog.Errorf("failed to init ipam from ovn eip cr %s: %v", oeip.Name, err)
}
oeips, err := c.ovnEipsLister.List(labels.Everything())
if err != nil {
klog.Errorf("failed to list ovn eips: %v", err)
return err
}
for _, oeip := range oeips {
if _, _, _, err = c.ipam.GetStaticAddress(oeip.Name, oeip.Name, oeip.Status.V4Ip, oeip.Status.MacAddress, oeip.Spec.ExternalSubnet, true); err != nil {
klog.Errorf("failed to init ipam from ovn eip cr %s: %v", oeip.Name, err)
}
}
nodes, err := c.nodesLister.List(labels.Everything())
Expand Down
8 changes: 8 additions & 0 deletions pkg/controller/ovn_eip.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -341,6 +341,14 @@ func (c *Controller) handleDelOvnEip(key string) error {
return err
}
}

if cachedEip.Spec.Type == util.LrpUsingEip {
if err := c.ovnLegacyClient.DeleteLogicalRouterPort(key); err != nil {
klog.Errorf("failed to delete lrp %s, %v", key, err)
return err
}
}

if err = c.handleDelOvnEipFinalizer(cachedEip); err != nil {
klog.Errorf("failed to remove finalizer from ovn eip, %v", err)
return err
Expand Down
Loading

0 comments on commit 5387acf

Please sign in to comment.