Skip to content

Commit

Permalink
add ovn DVR fip e2e (#2780)
Browse files Browse the repository at this point in the history 
* ovn fip e2e use DVR mode

* Update test/e2e/ovn-vpc-nat-gw/e2e_test.go

Co-authored-by: 张祖建 <zhangzujian.7@gmail.com>
  • Loading branch information
@bobz965 @zhangzujian
bobz965 and zhangzujian committed May 11, 2023
1 parent 0127e10 commit 1511573
Show file tree
Hide file tree
Showing 9 changed files with 209 additions and 120 deletions.
12 changes: 6 additions & 6 deletions pkg/controller/ovn_dnat.go
View file
Expand Up @@ -314,7 +314,6 @@ func (c *Controller) handleDelOvnDnatRule(key string) error {
klog.Errorf("failed to get eip, %v", err)
return err
}

if cachedDnat.Status.Vpc != "" && cachedDnat.Status.V4Eip != "" && cachedDnat.Status.ExternalPort != "" {
if err = c.DelDnatRule(cachedDnat.Status.Vpc, cachedDnat.Name,
cachedDnat.Status.V4Eip, cachedDnat.Status.ExternalPort); err != nil {
Expand All @@ -323,16 +322,17 @@ func (c *Controller) handleDelOvnDnatRule(key string) error {
}
}

if err = c.handleDelOvnDnatFinalizer(cachedDnat); err != nil {
klog.Errorf("failed to handle remove finalizer from ovn dnat, %v", err)
if err = c.handleDelOvnEipFinalizer(cachedEip, util.OvnDnatUseEipFinalizer); err != nil {
klog.Errorf("failed to handle remove finalizer from ovn eip, %v", err)
return err
}

c.resetOvnEipQueue.Add(cachedDnat.Spec.OvnEip)
if err = c.handleDelOvnEipFinalizer(cachedEip, util.OvnDnatUseEipFinalizer); err != nil {
klog.Errorf("failed to handle remove finalizer from ovn eip, %v", err)

if err = c.handleDelOvnDnatFinalizer(cachedDnat); err != nil {
klog.Errorf("failed to handle remove finalizer from ovn dnat, %v", err)
return err
}

return nil
}

Expand Down
71 changes: 36 additions & 35 deletions pkg/controller/ovn_eip.go
View file
Expand Up @@ -40,20 +40,20 @@ func (c *Controller) enqueueUpdateOvnEip(old, new interface{}) {
utilruntime.HandleError(err)
return
}
oldovnEip := old.(*kubeovnv1.OvnEip)
newovnEip := new.(*kubeovnv1.OvnEip)
if newovnEip.DeletionTimestamp != nil {
if len(newovnEip.Finalizers) == 0 {
oldEip := old.(*kubeovnv1.OvnEip)
newEip := new.(*kubeovnv1.OvnEip)
if newEip.DeletionTimestamp != nil {
if len(newEip.Finalizers) == 0 {
// avoid delete eip twice
return
} else {
klog.V(3).Infof("enqueue del ovn eip %s", key)
c.delOvnEipQueue.Add(key)
c.delOvnEipQueue.Add(newEip)
return
}
}
if oldovnEip.Spec.V4Ip != "" && oldovnEip.Spec.V4Ip != newovnEip.Spec.V4Ip ||
oldovnEip.Spec.MacAddress != "" && oldovnEip.Spec.MacAddress != newovnEip.Spec.MacAddress {
if oldEip.Spec.V4Ip != "" && oldEip.Spec.V4Ip != newEip.Spec.V4Ip ||
oldEip.Spec.MacAddress != "" && oldEip.Spec.MacAddress != newEip.Spec.MacAddress {
klog.Infof("not support change ip or mac for eip %s", key)
c.resetOvnEipQueue.Add(key)
return
Expand All @@ -70,7 +70,8 @@ func (c *Controller) enqueueDelOvnEip(obj interface{}) {
return
}
klog.V(3).Infof("enqueue del ovn eip %s", key)
c.delOvnEipQueue.Add(key)
eip := obj.(*kubeovnv1.OvnEip)
c.delOvnEipQueue.Add(eip)
}

func (c *Controller) runAddOvnEipWorker() {
Expand Down Expand Up @@ -184,16 +185,16 @@ func (c *Controller) processNextDeleteOvnEipWorkItem() bool {
}
err := func(obj interface{}) error {
defer c.delOvnEipQueue.Done(obj)
var key string
var eip *kubeovnv1.OvnEip
var ok bool
if key, ok = obj.(string); !ok {
if eip, ok = obj.(*kubeovnv1.OvnEip); !ok {
c.delOvnEipQueue.Forget(obj)
utilruntime.HandleError(fmt.Errorf("expected string in workqueue but got %#v", obj))
utilruntime.HandleError(fmt.Errorf("expected ovn eip in workqueue but got %#v", obj))
return nil
}
if err := c.handleDelOvnEip(key); err != nil {
c.delOvnEipQueue.AddRateLimited(key)
return fmt.Errorf("error syncing '%s': %s, requeuing", key, err.Error())
if err := c.handleDelOvnEip(eip); err != nil {
c.delOvnEipQueue.AddRateLimited(obj)
return fmt.Errorf("error syncing '%s': %s, requeuing", eip.Name, err.Error())
}
c.delOvnEipQueue.Forget(obj)
return nil
Expand Down Expand Up @@ -277,6 +278,7 @@ func (c *Controller) handleUpdateOvnEip(key string) error {
}
return err
}
klog.V(3).Infof("handle update ovn eip %s", cachedEip.Name)
if !cachedEip.DeletionTimestamp.IsZero() {
subnetName := cachedEip.Spec.ExternalSubnet
if subnetName == "" {
Expand Down Expand Up @@ -311,7 +313,11 @@ func (c *Controller) handleResetOvnEip(key string) error {
}
return err
}
if !cachedEip.DeletionTimestamp.IsZero() {
return nil
}

klog.V(3).Infof("handle reset ovn eip %s", cachedEip.Name)
var notUse bool
if notUse, err = c.isOvnEipNotUse(cachedEip); err != nil {
klog.Errorf("failed to check whether ovn eip '%s' is still in use, %v", cachedEip.Name, err)
Expand Down Expand Up @@ -341,37 +347,30 @@ func (c *Controller) handleResetOvnEip(key string) error {
return nil
}

func (c *Controller) handleDelOvnEip(key string) error {
klog.V(3).Infof("handle del ovn eip %s", key)
cachedEip, err := c.ovnEipsLister.Get(key)
if err != nil {
if k8serrors.IsNotFound(err) {
return nil
}
return err
}

if len(cachedEip.Finalizers) > 1 {
func (c *Controller) handleDelOvnEip(eip *kubeovnv1.OvnEip) error {
klog.V(3).Infof("handle del ovn eip %s", eip.Name)
if len(eip.Finalizers) > 1 {
err := errors.New("eip is referenced, it cannot be deleted directly")
klog.Errorf("failed to delete eip %s, %v", key, err)
klog.Errorf("failed to delete eip %s, %v", eip.Name, err)
return err
}

if cachedEip.Spec.Type == util.NodeExtGwUsingEip {
if err := c.ovnClient.DeleteLogicalSwitchPort(cachedEip.Name); err != nil {
klog.Errorf("failed to delete lsp %s, %v", cachedEip.Name, err)
if eip.Spec.Type == util.NodeExtGwUsingEip {
if err := c.ovnClient.DeleteLogicalSwitchPort(eip.Name); err != nil {
klog.Errorf("failed to delete lsp %s, %v", eip.Name, err)
return err
}
}

if cachedEip.Spec.Type == util.LrpUsingEip {
if err := c.ovnClient.DeleteLogicalRouterPort(key); err != nil {
klog.Errorf("failed to delete lrp %s, %v", key, err)
if eip.Spec.Type == util.LrpUsingEip {
if err := c.ovnClient.DeleteLogicalRouterPort(eip.Name); err != nil {
klog.Errorf("failed to delete lrp %s, %v", eip.Name, err)
return err
}
}

c.ipam.ReleaseAddressByPod(key)
c.ipam.ReleaseAddressByPod(eip.Name)
c.updateSubnetStatusQueue.Add(eip.Spec.ExternalSubnet)
return nil
}

Expand Down Expand Up @@ -719,7 +718,8 @@ func (c *Controller) isOvnEipNotUse(cachedEip *kubeovnv1.OvnEip) (bool, error) {
return false, err
}
for _, item := range dnats {
if item.Annotations[util.VpcEipAnnotation] == cachedEip.Name {
if item.DeletionTimestamp.IsZero() && item.Annotations[util.VpcEipAnnotation] == cachedEip.Name {
klog.Infof("ovn dnat %s is using eip %s, %v", item.Name, cachedEip.Name, err)
return false, nil
}
}
Expand All @@ -731,7 +731,8 @@ func (c *Controller) isOvnEipNotUse(cachedEip *kubeovnv1.OvnEip) (bool, error) {
return false, err
}
for _, item := range snats {
if item.Annotations[util.VpcEipAnnotation] == cachedEip.Name {
if item.DeletionTimestamp.IsZero() && item.Annotations[util.VpcEipAnnotation] == cachedEip.Name {
klog.Infof("ovn snat %s is using eip %s, %v", item.Name, cachedEip.Name, err)
return false, nil
}
}
Expand Down
9 changes: 5 additions & 4 deletions pkg/controller/ovn_fip.go
View file
Expand Up @@ -399,14 +399,15 @@ func (c *Controller) handleDelOvnFip(key string) error {
return err
}
}
if err = c.handleDelOvnFipFinalizer(cachedFip); err != nil {
klog.Errorf("failed to handle remove finalizer from ovn fip, %v", err)
if err = c.handleDelOvnEipFinalizer(cachedEip, util.OvnFipUseEipFinalizer); err != nil {
klog.Errorf("failed to handle remove finalizer from ovn eip, %v", err)
return err
}
// reset eip
c.resetOvnEipQueue.Add(cachedFip.Spec.OvnEip)
if err = c.handleDelOvnEipFinalizer(cachedEip, util.OvnFipUseEipFinalizer); err != nil {
klog.Errorf("failed to handle remove finalizer from ovn eip, %v", err)

if err = c.handleDelOvnFipFinalizer(cachedFip); err != nil {
klog.Errorf("failed to handle remove finalizer from ovn fip, %v", err)
return err
}
return nil
Expand Down
11 changes: 6 additions & 5 deletions pkg/controller/ovn_snat.go
View file
Expand Up @@ -283,22 +283,23 @@ func (c *Controller) handleUpdateOvnSnatRule(key string) error {
}
// should delete
if !cachedSnat.DeletionTimestamp.IsZero() {
klog.V(3).Infof("ovn clean snat %s", key)
klog.V(3).Infof("ovn delete snat %s", key)
// ovn delete snat
if cachedSnat.Status.Vpc != "" && cachedSnat.Status.V4Eip != "" && cachedSnat.Status.V4IpCidr != "" {
if err = c.ovnLegacyClient.DeleteSnatRule(cachedSnat.Status.Vpc, cachedSnat.Status.V4Eip, cachedSnat.Status.V4IpCidr); err != nil {
klog.Errorf("failed to delete snat, %v", err)
return err
}
}
if err = c.handleDelOvnSnatRuleFinalizer(cachedSnat); err != nil {
klog.Errorf("failed to handle finalizer for snat %s, %v", key, err)
if err = c.handleDelOvnEipFinalizer(cachedEip, util.OvnSnatUseEipFinalizer); err != nil {
klog.Errorf("failed to handle finalizer for eip %s, %v", key, err)
return err
}
// reset eip
c.resetOvnEipQueue.Add(cachedSnat.Spec.OvnEip)
if err = c.handleDelOvnEipFinalizer(cachedEip, util.OvnSnatUseEipFinalizer); err != nil {
klog.Errorf("failed to handle finalizer for eip %s, %v", key, err)

if err = c.handleDelOvnSnatRuleFinalizer(cachedSnat); err != nil {
klog.Errorf("failed to handle finalizer for snat %s, %v", key, err)
return err
}
return nil
Expand Down
27 changes: 15 additions & 12 deletions pkg/controller/vip.go
View file
Expand Up @@ -28,6 +28,7 @@ func (c *Controller) enqueueAddVirtualIp(obj interface{}) {
utilruntime.HandleError(err)
return
}
klog.V(3).Infof("enqueue add vip %s", key)
c.addVirtualIpQueue.Add(key)
}

Expand All @@ -45,6 +46,7 @@ func (c *Controller) enqueueUpdateVirtualIp(old, new interface{}) {
oldVip.Spec.ParentMac != newVip.Spec.ParentMac ||
oldVip.Spec.ParentV4ip != newVip.Spec.ParentV4ip ||
oldVip.Spec.V4ip != newVip.Spec.V4ip {
klog.V(3).Infof("enqueue update vip %s", key)
c.updateVirtualIpQueue.Add(key)
}
}
Expand All @@ -56,9 +58,9 @@ func (c *Controller) enqueueDelVirtualIp(obj interface{}) {
utilruntime.HandleError(err)
return
}
c.delVirtualIpQueue.Add(key)
vipObj := obj.(*kubeovnv1.Vip)
c.updateSubnetStatusQueue.Add(vipObj.Spec.Subnet)
klog.V(3).Infof("enqueue del vip %s", key)
vip := obj.(*kubeovnv1.Vip)
c.delVirtualIpQueue.Add(vip)
}

func (c *Controller) runAddVirtualIpWorker() {
Expand Down Expand Up @@ -142,16 +144,16 @@ func (c *Controller) processNextDeleteVirtualIpWorkItem() bool {

err := func(obj interface{}) error {
defer c.delVirtualIpQueue.Done(obj)
var key string
var vip *kubeovnv1.Vip
var ok bool
if key, ok = obj.(string); !ok {
if vip, ok = obj.(*kubeovnv1.Vip); !ok {
c.delVirtualIpQueue.Forget(obj)
utilruntime.HandleError(fmt.Errorf("expected string in workqueue but got %#v", obj))
utilruntime.HandleError(fmt.Errorf("expected vip in workqueue but got %#v", obj))
return nil
}
if err := c.handleDelVirtualIp(key); err != nil {
c.delVirtualIpQueue.AddRateLimited(key)
return fmt.Errorf("error syncing '%s': %s, requeuing", key, err.Error())
if err := c.handleDelVirtualIp(vip); err != nil {
c.delVirtualIpQueue.AddRateLimited(obj)
return fmt.Errorf("error syncing '%s': %s, requeuing", vip.Name, err.Error())
}
c.delVirtualIpQueue.Forget(obj)
return nil
Expand Down Expand Up @@ -261,9 +263,10 @@ func (c *Controller) handleUpdateVirtualIp(key string) error {
return nil
}

func (c *Controller) handleDelVirtualIp(key string) error {
klog.V(3).Infof("release vip %s", key)
c.ipam.ReleaseAddressByPod(key)
func (c *Controller) handleDelVirtualIp(vip *kubeovnv1.Vip) error {
klog.V(3).Infof("delete vip %s", vip.Name)
c.ipam.ReleaseAddressByPod(vip.Name)
c.updateSubnetStatusQueue.Add(vip.Spec.Subnet)
return nil
}

Expand Down
8 changes: 7 additions & 1 deletion pkg/ovs/ovn-nb.go
View file
Expand Up @@ -27,8 +27,14 @@ const (
func (c *ovnClient) CreateGatewayLogicalSwitch(lsName, lrName, provider, ip, mac string, vlanID int, chassises ...string) error {
lspName := fmt.Sprintf("%s-%s", lsName, lrName)
lrpName := fmt.Sprintf("%s-%s", lrName, lsName)
localnetLspName := fmt.Sprintf("ln-%s", lsName)

// delete old localnet lsp when upgrade before v1.12
oldLocalnetLspName := fmt.Sprintf("ln-%s", lsName)
if err := c.DeleteLogicalSwitchPort(oldLocalnetLspName); err != nil {
return fmt.Errorf("failed to delete old localnet %s: %v", oldLocalnetLspName, err)
}

localnetLspName := GetLocalnetName(lsName)
if err := c.CreateBareLogicalSwitch(lsName); err != nil {
return fmt.Errorf("create logical switch %s: %v", lsName, err)
}
Expand Down
2 changes: 1 addition & 1 deletion pkg/ovs/ovn-nb_test.go
View file
Expand Up @@ -19,7 +19,7 @@ func (suite *OvnClientTestSuite) testCreateGatewayLogicalSwitch() {
lrName := "test-create-gw-lr"
lspName := fmt.Sprintf("%s-%s", lsName, lrName)
lrpName := fmt.Sprintf("%s-%s", lrName, lsName)
localnetLspName := fmt.Sprintf("ln-%s", lsName)
localnetLspName := GetLocalnetName(lsName)
chassises := []string{"5de32fcb-495a-40df-919e-f09812c4d11e", "25310674-65ce-69fd-bcfa-65b25268926b"}

err := ovnClient.CreateLogicalRouter(lrName)
Expand Down
6 changes: 3 additions & 3 deletions test/e2e/framework/expect.go
View file
Expand Up @@ -67,7 +67,7 @@ func ExpectNoErrorWithOffset(offset int, err error, explain ...interface{}) {
}
}

// This intentionally doesn't use gomega.Expect. Instead we take
// This intentionally doesn't use gomega.Expect. Instead，we take
// full control over what information is presented where:
// - The complete error object is logged because it may contain
// additional information that isn't included in its error
Expand Down Expand Up @@ -148,7 +148,7 @@ func ExpectFalse(actual interface{}, explain ...interface{}) {
gomega.ExpectWithOffset(1, actual).NotTo(gomega.BeTrue(), explain...)
}

// ExpectZero expects actual actual is the zero value for its type or actual is nil.
// ExpectZero expects actual is the zero value for its type or actual is nil.
func ExpectZero(actual interface{}, explain ...interface{}) {
gomega.ExpectWithOffset(1, actual).To(gomega.BeZero(), explain...)
}
Expand All @@ -158,7 +158,7 @@ func ExpectNotZero(actual interface{}, explain ...interface{}) {
gomega.ExpectWithOffset(1, actual).NotTo(gomega.BeZero(), explain...)
}

// ExpectUUID expects that the given string is an UUID.
// ExpectUUID expects that the given string is a UUID.
func ExpectUUID(s string) {
ginkgo.By("verifying the string " + s + " is an UUID")
ExpectTrue(uuidRegex.MatchString(s))
Expand Down

0 comments on commit 1511573

Please sign in to comment.