diff --git a/go.mod b/go.mod index e4eff95245a..de7d928c825 100644 --- a/go.mod +++ b/go.mod @@ -35,6 +35,7 @@ require ( github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.8.4 github.com/vishvananda/netlink v1.2.1-beta.2 + golang.org/x/exp v0.0.0-20231006140011-7918f672742d golang.org/x/mod v0.13.0 golang.org/x/sys v0.13.0 golang.org/x/time v0.3.0 diff --git a/mocks/pkg/ovs/interface.go b/mocks/pkg/ovs/interface.go index 96161b7424c..ab5baa1ba4a 100644 --- a/mocks/pkg/ovs/interface.go +++ b/mocks/pkg/ovs/interface.go @@ -935,6 +935,34 @@ func (mr *MockLoadBalancerMockRecorder) ListLoadBalancers(filter interface{}) *g return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListLoadBalancers", reflect.TypeOf((*MockLoadBalancer)(nil).ListLoadBalancers), filter) } +// LoadBalancerAddHealthCheck mocks base method. +func (m *MockLoadBalancer) LoadBalancerAddHealthCheck(lbName, vip string, ignoreHealthCheck bool, ipPortMapping, externals map[string]string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "LoadBalancerAddHealthCheck", lbName, vip, ignoreHealthCheck, ipPortMapping, externals) + ret0, _ := ret[0].(error) + return ret0 +} + +// LoadBalancerAddHealthCheck indicates an expected call of LoadBalancerAddHealthCheck. +func (mr *MockLoadBalancerMockRecorder) LoadBalancerAddHealthCheck(lbName, vip, ignoreHealthCheck, ipPortMapping, externals interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "LoadBalancerAddHealthCheck", reflect.TypeOf((*MockLoadBalancer)(nil).LoadBalancerAddHealthCheck), lbName, vip, ignoreHealthCheck, ipPortMapping, externals) +} + +// LoadBalancerAddIPPortMapping mocks base method. +func (m *MockLoadBalancer) LoadBalancerAddIPPortMapping(lbName, vip string, ipPortMappings map[string]string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "LoadBalancerAddIPPortMapping", lbName, vip, ipPortMappings) + ret0, _ := ret[0].(error) + return ret0 +} + +// LoadBalancerAddIPPortMapping indicates an expected call of LoadBalancerAddIPPortMapping. +func (mr *MockLoadBalancerMockRecorder) LoadBalancerAddIPPortMapping(lbName, vip, ipPortMappings interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "LoadBalancerAddIPPortMapping", reflect.TypeOf((*MockLoadBalancer)(nil).LoadBalancerAddIPPortMapping), lbName, vip, ipPortMappings) +} + // LoadBalancerAddVip mocks base method. func (m *MockLoadBalancer) LoadBalancerAddVip(lbName, vip string, backends ...string) error { m.ctrl.T.Helper() @@ -954,18 +982,46 @@ func (mr *MockLoadBalancerMockRecorder) LoadBalancerAddVip(lbName, vip interface return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "LoadBalancerAddVip", reflect.TypeOf((*MockLoadBalancer)(nil).LoadBalancerAddVip), varargs...) } +// LoadBalancerDeleteHealthCheck mocks base method. +func (m *MockLoadBalancer) LoadBalancerDeleteHealthCheck(lbName, uuid string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "LoadBalancerDeleteHealthCheck", lbName, uuid) + ret0, _ := ret[0].(error) + return ret0 +} + +// LoadBalancerDeleteHealthCheck indicates an expected call of LoadBalancerDeleteHealthCheck. +func (mr *MockLoadBalancerMockRecorder) LoadBalancerDeleteHealthCheck(lbName, uuid interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "LoadBalancerDeleteHealthCheck", reflect.TypeOf((*MockLoadBalancer)(nil).LoadBalancerDeleteHealthCheck), lbName, uuid) +} + +// LoadBalancerDeleteIPPortMapping mocks base method. +func (m *MockLoadBalancer) LoadBalancerDeleteIPPortMapping(lbName, vip string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "LoadBalancerDeleteIPPortMapping", lbName, vip) + ret0, _ := ret[0].(error) + return ret0 +} + +// LoadBalancerDeleteIPPortMapping indicates an expected call of LoadBalancerDeleteIPPortMapping. +func (mr *MockLoadBalancerMockRecorder) LoadBalancerDeleteIPPortMapping(lbName, vip interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "LoadBalancerDeleteIPPortMapping", reflect.TypeOf((*MockLoadBalancer)(nil).LoadBalancerDeleteIPPortMapping), lbName, vip) +} + // LoadBalancerDeleteVip mocks base method. -func (m *MockLoadBalancer) LoadBalancerDeleteVip(lbName, vip string) error { +func (m *MockLoadBalancer) LoadBalancerDeleteVip(lbName, vip string, ignoreHealthCheck bool) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "LoadBalancerDeleteVip", lbName, vip) + ret := m.ctrl.Call(m, "LoadBalancerDeleteVip", lbName, vip, ignoreHealthCheck) ret0, _ := ret[0].(error) return ret0 } // LoadBalancerDeleteVip indicates an expected call of LoadBalancerDeleteVip. -func (mr *MockLoadBalancerMockRecorder) LoadBalancerDeleteVip(lbName, vip interface{}) *gomock.Call { +func (mr *MockLoadBalancerMockRecorder) LoadBalancerDeleteVip(lbName, vip, ignoreHealthCheck interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "LoadBalancerDeleteVip", reflect.TypeOf((*MockLoadBalancer)(nil).LoadBalancerDeleteVip), lbName, vip) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "LoadBalancerDeleteVip", reflect.TypeOf((*MockLoadBalancer)(nil).LoadBalancerDeleteVip), lbName, vip, ignoreHealthCheck) } // LoadBalancerExists mocks base method. @@ -983,6 +1039,20 @@ func (mr *MockLoadBalancerMockRecorder) LoadBalancerExists(lbName interface{}) * return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "LoadBalancerExists", reflect.TypeOf((*MockLoadBalancer)(nil).LoadBalancerExists), lbName) } +// LoadBalancerUpdateIPPortMapping mocks base method. +func (m *MockLoadBalancer) LoadBalancerUpdateIPPortMapping(lbName, vip string, ipPortMappings map[string]string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "LoadBalancerUpdateIPPortMapping", lbName, vip, ipPortMappings) + ret0, _ := ret[0].(error) + return ret0 +} + +// LoadBalancerUpdateIPPortMapping indicates an expected call of LoadBalancerUpdateIPPortMapping. +func (mr *MockLoadBalancerMockRecorder) LoadBalancerUpdateIPPortMapping(lbName, vip, ipPortMappings interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "LoadBalancerUpdateIPPortMapping", reflect.TypeOf((*MockLoadBalancer)(nil).LoadBalancerUpdateIPPortMapping), lbName, vip, ipPortMappings) +} + // SetLoadBalancerAffinityTimeout mocks base method. func (m *MockLoadBalancer) SetLoadBalancerAffinityTimeout(lbName string, timeout int) error { m.ctrl.T.Helper() @@ -997,6 +1067,131 @@ func (mr *MockLoadBalancerMockRecorder) SetLoadBalancerAffinityTimeout(lbName, t return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetLoadBalancerAffinityTimeout", reflect.TypeOf((*MockLoadBalancer)(nil).SetLoadBalancerAffinityTimeout), lbName, timeout) } +// MockLoadBalancerHealthCheck is a mock of LoadBalancerHealthCheck interface. +type MockLoadBalancerHealthCheck struct { + ctrl *gomock.Controller + recorder *MockLoadBalancerHealthCheckMockRecorder +} + +// MockLoadBalancerHealthCheckMockRecorder is the mock recorder for MockLoadBalancerHealthCheck. +type MockLoadBalancerHealthCheckMockRecorder struct { + mock *MockLoadBalancerHealthCheck +} + +// NewMockLoadBalancerHealthCheck creates a new mock instance. +func NewMockLoadBalancerHealthCheck(ctrl *gomock.Controller) *MockLoadBalancerHealthCheck { + mock := &MockLoadBalancerHealthCheck{ctrl: ctrl} + mock.recorder = &MockLoadBalancerHealthCheckMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockLoadBalancerHealthCheck) EXPECT() *MockLoadBalancerHealthCheckMockRecorder { + return m.recorder +} + +// AddLoadBalancerHealthCheck mocks base method. +func (m *MockLoadBalancerHealthCheck) AddLoadBalancerHealthCheck(lbName, vip string, externals map[string]string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AddLoadBalancerHealthCheck", lbName, vip, externals) + ret0, _ := ret[0].(error) + return ret0 +} + +// AddLoadBalancerHealthCheck indicates an expected call of AddLoadBalancerHealthCheck. +func (mr *MockLoadBalancerHealthCheckMockRecorder) AddLoadBalancerHealthCheck(lbName, vip, externals interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddLoadBalancerHealthCheck", reflect.TypeOf((*MockLoadBalancerHealthCheck)(nil).AddLoadBalancerHealthCheck), lbName, vip, externals) +} + +// CreateLoadBalancerHealthCheck mocks base method. +func (m *MockLoadBalancerHealthCheck) CreateLoadBalancerHealthCheck(lbName, vip string, lbhc *ovnnb.LoadBalancerHealthCheck) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CreateLoadBalancerHealthCheck", lbName, vip, lbhc) + ret0, _ := ret[0].(error) + return ret0 +} + +// CreateLoadBalancerHealthCheck indicates an expected call of CreateLoadBalancerHealthCheck. +func (mr *MockLoadBalancerHealthCheckMockRecorder) CreateLoadBalancerHealthCheck(lbName, vip, lbhc interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateLoadBalancerHealthCheck", reflect.TypeOf((*MockLoadBalancerHealthCheck)(nil).CreateLoadBalancerHealthCheck), lbName, vip, lbhc) +} + +// DeleteLoadBalancerHealthCheck mocks base method. +func (m *MockLoadBalancerHealthCheck) DeleteLoadBalancerHealthCheck(lbName, vip string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteLoadBalancerHealthCheck", lbName, vip) + ret0, _ := ret[0].(error) + return ret0 +} + +// DeleteLoadBalancerHealthCheck indicates an expected call of DeleteLoadBalancerHealthCheck. +func (mr *MockLoadBalancerHealthCheckMockRecorder) DeleteLoadBalancerHealthCheck(lbName, vip interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteLoadBalancerHealthCheck", reflect.TypeOf((*MockLoadBalancerHealthCheck)(nil).DeleteLoadBalancerHealthCheck), lbName, vip) +} + +// DeleteLoadBalancerHealthChecks mocks base method. +func (m *MockLoadBalancerHealthCheck) DeleteLoadBalancerHealthChecks(filter func(*ovnnb.LoadBalancerHealthCheck) bool) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteLoadBalancerHealthChecks", filter) + ret0, _ := ret[0].(error) + return ret0 +} + +// DeleteLoadBalancerHealthChecks indicates an expected call of DeleteLoadBalancerHealthChecks. +func (mr *MockLoadBalancerHealthCheckMockRecorder) DeleteLoadBalancerHealthChecks(filter interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteLoadBalancerHealthChecks", reflect.TypeOf((*MockLoadBalancerHealthCheck)(nil).DeleteLoadBalancerHealthChecks), filter) +} + +// GetLoadBalancerHealthCheck mocks base method. +func (m *MockLoadBalancerHealthCheck) GetLoadBalancerHealthCheck(lbName, vip string, ignoreNotFound bool) (*ovnnb.LoadBalancer, *ovnnb.LoadBalancerHealthCheck, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetLoadBalancerHealthCheck", lbName, vip, ignoreNotFound) + ret0, _ := ret[0].(*ovnnb.LoadBalancer) + ret1, _ := ret[1].(*ovnnb.LoadBalancerHealthCheck) + ret2, _ := ret[2].(error) + return ret0, ret1, ret2 +} + +// GetLoadBalancerHealthCheck indicates an expected call of GetLoadBalancerHealthCheck. +func (mr *MockLoadBalancerHealthCheckMockRecorder) GetLoadBalancerHealthCheck(lbName, vip, ignoreNotFound interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetLoadBalancerHealthCheck", reflect.TypeOf((*MockLoadBalancerHealthCheck)(nil).GetLoadBalancerHealthCheck), lbName, vip, ignoreNotFound) +} + +// ListLoadBalancerHealthChecks mocks base method. +func (m *MockLoadBalancerHealthCheck) ListLoadBalancerHealthChecks(filter func(*ovnnb.LoadBalancerHealthCheck) bool) ([]ovnnb.LoadBalancerHealthCheck, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListLoadBalancerHealthChecks", filter) + ret0, _ := ret[0].([]ovnnb.LoadBalancerHealthCheck) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListLoadBalancerHealthChecks indicates an expected call of ListLoadBalancerHealthChecks. +func (mr *MockLoadBalancerHealthCheckMockRecorder) ListLoadBalancerHealthChecks(filter interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListLoadBalancerHealthChecks", reflect.TypeOf((*MockLoadBalancerHealthCheck)(nil).ListLoadBalancerHealthChecks), filter) +} + +// LoadBalancerHealthCheckExists mocks base method. +func (m *MockLoadBalancerHealthCheck) LoadBalancerHealthCheckExists(lbName, vip string) (bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "LoadBalancerHealthCheckExists", lbName, vip) + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// LoadBalancerHealthCheckExists indicates an expected call of LoadBalancerHealthCheckExists. +func (mr *MockLoadBalancerHealthCheckMockRecorder) LoadBalancerHealthCheckExists(lbName, vip interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "LoadBalancerHealthCheckExists", reflect.TypeOf((*MockLoadBalancerHealthCheck)(nil).LoadBalancerHealthCheckExists), lbName, vip) +} + // MockPortGroup is a mock of PortGroup interface. type MockPortGroup struct { ctrl *gomock.Controller @@ -1936,6 +2131,20 @@ func (m *MockNbClient) EXPECT() *MockNbClientMockRecorder { return m.recorder } +// AddLoadBalancerHealthCheck mocks base method. +func (m *MockNbClient) AddLoadBalancerHealthCheck(lbName, vip string, externals map[string]string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AddLoadBalancerHealthCheck", lbName, vip, externals) + ret0, _ := ret[0].(error) + return ret0 +} + +// AddLoadBalancerHealthCheck indicates an expected call of AddLoadBalancerHealthCheck. +func (mr *MockNbClientMockRecorder) AddLoadBalancerHealthCheck(lbName, vip, externals interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddLoadBalancerHealthCheck", reflect.TypeOf((*MockNbClient)(nil).AddLoadBalancerHealthCheck), lbName, vip, externals) +} + // AddLogicalRouterPolicy mocks base method. func (m *MockNbClient) AddLogicalRouterPolicy(lrName string, priority int, match, action string, nextHops []string, externalIDs map[string]string) error { m.ctrl.T.Helper() @@ -2134,6 +2343,20 @@ func (mr *MockNbClientMockRecorder) CreateLoadBalancer(lbName, protocol, selectF return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateLoadBalancer", reflect.TypeOf((*MockNbClient)(nil).CreateLoadBalancer), lbName, protocol, selectFields) } +// CreateLoadBalancerHealthCheck mocks base method. +func (m *MockNbClient) CreateLoadBalancerHealthCheck(lbName, vip string, lbhc *ovnnb.LoadBalancerHealthCheck) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CreateLoadBalancerHealthCheck", lbName, vip, lbhc) + ret0, _ := ret[0].(error) + return ret0 +} + +// CreateLoadBalancerHealthCheck indicates an expected call of CreateLoadBalancerHealthCheck. +func (mr *MockNbClientMockRecorder) CreateLoadBalancerHealthCheck(lbName, vip, lbhc interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateLoadBalancerHealthCheck", reflect.TypeOf((*MockNbClient)(nil).CreateLoadBalancerHealthCheck), lbName, vip, lbhc) +} + // CreateLocalnetLogicalSwitchPort mocks base method. func (m *MockNbClient) CreateLocalnetLogicalSwitchPort(lsName, lspName, provider string, vlanID int) error { m.ctrl.T.Helper() @@ -2415,6 +2638,34 @@ func (mr *MockNbClientMockRecorder) DeleteDHCPOptionsByUUIDs(uuidList ...interfa return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteDHCPOptionsByUUIDs", reflect.TypeOf((*MockNbClient)(nil).DeleteDHCPOptionsByUUIDs), uuidList...) } +// DeleteLoadBalancerHealthCheck mocks base method. +func (m *MockNbClient) DeleteLoadBalancerHealthCheck(lbName, vip string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteLoadBalancerHealthCheck", lbName, vip) + ret0, _ := ret[0].(error) + return ret0 +} + +// DeleteLoadBalancerHealthCheck indicates an expected call of DeleteLoadBalancerHealthCheck. +func (mr *MockNbClientMockRecorder) DeleteLoadBalancerHealthCheck(lbName, vip interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteLoadBalancerHealthCheck", reflect.TypeOf((*MockNbClient)(nil).DeleteLoadBalancerHealthCheck), lbName, vip) +} + +// DeleteLoadBalancerHealthChecks mocks base method. +func (m *MockNbClient) DeleteLoadBalancerHealthChecks(filter func(*ovnnb.LoadBalancerHealthCheck) bool) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteLoadBalancerHealthChecks", filter) + ret0, _ := ret[0].(error) + return ret0 +} + +// DeleteLoadBalancerHealthChecks indicates an expected call of DeleteLoadBalancerHealthChecks. +func (mr *MockNbClientMockRecorder) DeleteLoadBalancerHealthChecks(filter interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteLoadBalancerHealthChecks", reflect.TypeOf((*MockNbClient)(nil).DeleteLoadBalancerHealthChecks), filter) +} + // DeleteLoadBalancers mocks base method. func (m *MockNbClient) DeleteLoadBalancers(filter func(*ovnnb.LoadBalancer) bool) error { m.ctrl.T.Helper() @@ -2682,6 +2933,22 @@ func (mr *MockNbClientMockRecorder) GetLoadBalancer(lbName, ignoreNotFound inter return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetLoadBalancer", reflect.TypeOf((*MockNbClient)(nil).GetLoadBalancer), lbName, ignoreNotFound) } +// GetLoadBalancerHealthCheck mocks base method. +func (m *MockNbClient) GetLoadBalancerHealthCheck(lbName, vip string, ignoreNotFound bool) (*ovnnb.LoadBalancer, *ovnnb.LoadBalancerHealthCheck, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetLoadBalancerHealthCheck", lbName, vip, ignoreNotFound) + ret0, _ := ret[0].(*ovnnb.LoadBalancer) + ret1, _ := ret[1].(*ovnnb.LoadBalancerHealthCheck) + ret2, _ := ret[2].(error) + return ret0, ret1, ret2 +} + +// GetLoadBalancerHealthCheck indicates an expected call of GetLoadBalancerHealthCheck. +func (mr *MockNbClientMockRecorder) GetLoadBalancerHealthCheck(lbName, vip, ignoreNotFound interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetLoadBalancerHealthCheck", reflect.TypeOf((*MockNbClient)(nil).GetLoadBalancerHealthCheck), lbName, vip, ignoreNotFound) +} + // GetLogicalRouter mocks base method. func (m *MockNbClient) GetLogicalRouter(lrName string, ignoreNotFound bool) (*ovnnb.LogicalRouter, error) { m.ctrl.T.Helper() @@ -2832,6 +3099,21 @@ func (mr *MockNbClientMockRecorder) ListDHCPOptions(needVendorFilter, externalID return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListDHCPOptions", reflect.TypeOf((*MockNbClient)(nil).ListDHCPOptions), needVendorFilter, externalIDs) } +// ListLoadBalancerHealthChecks mocks base method. +func (m *MockNbClient) ListLoadBalancerHealthChecks(filter func(*ovnnb.LoadBalancerHealthCheck) bool) ([]ovnnb.LoadBalancerHealthCheck, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListLoadBalancerHealthChecks", filter) + ret0, _ := ret[0].([]ovnnb.LoadBalancerHealthCheck) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListLoadBalancerHealthChecks indicates an expected call of ListLoadBalancerHealthChecks. +func (mr *MockNbClientMockRecorder) ListLoadBalancerHealthChecks(filter interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListLoadBalancerHealthChecks", reflect.TypeOf((*MockNbClient)(nil).ListLoadBalancerHealthChecks), filter) +} + // ListLoadBalancers mocks base method. func (m *MockNbClient) ListLoadBalancers(filter func(*ovnnb.LoadBalancer) bool) ([]ovnnb.LoadBalancer, error) { m.ctrl.T.Helper() @@ -3012,6 +3294,34 @@ func (mr *MockNbClientMockRecorder) ListPortGroups(externalIDs interface{}) *gom return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListPortGroups", reflect.TypeOf((*MockNbClient)(nil).ListPortGroups), externalIDs) } +// LoadBalancerAddHealthCheck mocks base method. +func (m *MockNbClient) LoadBalancerAddHealthCheck(lbName, vip string, ignoreHealthCheck bool, ipPortMapping, externals map[string]string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "LoadBalancerAddHealthCheck", lbName, vip, ignoreHealthCheck, ipPortMapping, externals) + ret0, _ := ret[0].(error) + return ret0 +} + +// LoadBalancerAddHealthCheck indicates an expected call of LoadBalancerAddHealthCheck. +func (mr *MockNbClientMockRecorder) LoadBalancerAddHealthCheck(lbName, vip, ignoreHealthCheck, ipPortMapping, externals interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "LoadBalancerAddHealthCheck", reflect.TypeOf((*MockNbClient)(nil).LoadBalancerAddHealthCheck), lbName, vip, ignoreHealthCheck, ipPortMapping, externals) +} + +// LoadBalancerAddIPPortMapping mocks base method. +func (m *MockNbClient) LoadBalancerAddIPPortMapping(lbName, vip string, ipPortMappings map[string]string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "LoadBalancerAddIPPortMapping", lbName, vip, ipPortMappings) + ret0, _ := ret[0].(error) + return ret0 +} + +// LoadBalancerAddIPPortMapping indicates an expected call of LoadBalancerAddIPPortMapping. +func (mr *MockNbClientMockRecorder) LoadBalancerAddIPPortMapping(lbName, vip, ipPortMappings interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "LoadBalancerAddIPPortMapping", reflect.TypeOf((*MockNbClient)(nil).LoadBalancerAddIPPortMapping), lbName, vip, ipPortMappings) +} + // LoadBalancerAddVip mocks base method. func (m *MockNbClient) LoadBalancerAddVip(lbName, vip string, backends ...string) error { m.ctrl.T.Helper() @@ -3031,18 +3341,46 @@ func (mr *MockNbClientMockRecorder) LoadBalancerAddVip(lbName, vip interface{}, return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "LoadBalancerAddVip", reflect.TypeOf((*MockNbClient)(nil).LoadBalancerAddVip), varargs...) } +// LoadBalancerDeleteHealthCheck mocks base method. +func (m *MockNbClient) LoadBalancerDeleteHealthCheck(lbName, uuid string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "LoadBalancerDeleteHealthCheck", lbName, uuid) + ret0, _ := ret[0].(error) + return ret0 +} + +// LoadBalancerDeleteHealthCheck indicates an expected call of LoadBalancerDeleteHealthCheck. +func (mr *MockNbClientMockRecorder) LoadBalancerDeleteHealthCheck(lbName, uuid interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "LoadBalancerDeleteHealthCheck", reflect.TypeOf((*MockNbClient)(nil).LoadBalancerDeleteHealthCheck), lbName, uuid) +} + +// LoadBalancerDeleteIPPortMapping mocks base method. +func (m *MockNbClient) LoadBalancerDeleteIPPortMapping(lbName, vip string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "LoadBalancerDeleteIPPortMapping", lbName, vip) + ret0, _ := ret[0].(error) + return ret0 +} + +// LoadBalancerDeleteIPPortMapping indicates an expected call of LoadBalancerDeleteIPPortMapping. +func (mr *MockNbClientMockRecorder) LoadBalancerDeleteIPPortMapping(lbName, vip interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "LoadBalancerDeleteIPPortMapping", reflect.TypeOf((*MockNbClient)(nil).LoadBalancerDeleteIPPortMapping), lbName, vip) +} + // LoadBalancerDeleteVip mocks base method. -func (m *MockNbClient) LoadBalancerDeleteVip(lbName, vip string) error { +func (m *MockNbClient) LoadBalancerDeleteVip(lbName, vip string, ignoreHealthCheck bool) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "LoadBalancerDeleteVip", lbName, vip) + ret := m.ctrl.Call(m, "LoadBalancerDeleteVip", lbName, vip, ignoreHealthCheck) ret0, _ := ret[0].(error) return ret0 } // LoadBalancerDeleteVip indicates an expected call of LoadBalancerDeleteVip. -func (mr *MockNbClientMockRecorder) LoadBalancerDeleteVip(lbName, vip interface{}) *gomock.Call { +func (mr *MockNbClientMockRecorder) LoadBalancerDeleteVip(lbName, vip, ignoreHealthCheck interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "LoadBalancerDeleteVip", reflect.TypeOf((*MockNbClient)(nil).LoadBalancerDeleteVip), lbName, vip) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "LoadBalancerDeleteVip", reflect.TypeOf((*MockNbClient)(nil).LoadBalancerDeleteVip), lbName, vip, ignoreHealthCheck) } // LoadBalancerExists mocks base method. @@ -3060,6 +3398,35 @@ func (mr *MockNbClientMockRecorder) LoadBalancerExists(lbName interface{}) *gomo return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "LoadBalancerExists", reflect.TypeOf((*MockNbClient)(nil).LoadBalancerExists), lbName) } +// LoadBalancerHealthCheckExists mocks base method. +func (m *MockNbClient) LoadBalancerHealthCheckExists(lbName, vip string) (bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "LoadBalancerHealthCheckExists", lbName, vip) + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// LoadBalancerHealthCheckExists indicates an expected call of LoadBalancerHealthCheckExists. +func (mr *MockNbClientMockRecorder) LoadBalancerHealthCheckExists(lbName, vip interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "LoadBalancerHealthCheckExists", reflect.TypeOf((*MockNbClient)(nil).LoadBalancerHealthCheckExists), lbName, vip) +} + +// LoadBalancerUpdateIPPortMapping mocks base method. +func (m *MockNbClient) LoadBalancerUpdateIPPortMapping(lbName, vip string, ipPortMappings map[string]string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "LoadBalancerUpdateIPPortMapping", lbName, vip, ipPortMappings) + ret0, _ := ret[0].(error) + return ret0 +} + +// LoadBalancerUpdateIPPortMapping indicates an expected call of LoadBalancerUpdateIPPortMapping. +func (mr *MockNbClientMockRecorder) LoadBalancerUpdateIPPortMapping(lbName, vip, ipPortMappings interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "LoadBalancerUpdateIPPortMapping", reflect.TypeOf((*MockNbClient)(nil).LoadBalancerUpdateIPPortMapping), lbName, vip, ipPortMappings) +} + // LogicalRouterExists mocks base method. func (m *MockNbClient) LogicalRouterExists(name string) (bool, error) { m.ctrl.T.Helper() diff --git a/pkg/controller/endpoint.go b/pkg/controller/endpoint.go index ed51d477b6b..818266efa60 100644 --- a/pkg/controller/endpoint.go +++ b/pkg/controller/endpoint.go @@ -12,12 +12,16 @@ import ( "k8s.io/client-go/tools/cache" "k8s.io/klog/v2" + kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1" "github.com/kubeovn/kube-ovn/pkg/util" ) func (c *Controller) enqueueAddEndpoint(obj interface{}) { - var key string - var err error + var ( + key string + err error + ) + if key, err = cache.MetaNamespaceKeyFunc(obj); err != nil { utilruntime.HandleError(err) return @@ -37,8 +41,11 @@ func (c *Controller) enqueueUpdateEndpoint(oldObj, newObj interface{}) { return } - var key string - var err error + var ( + key string + err error + ) + if key, err = cache.MetaNamespaceKeyFunc(newObj); err != nil { utilruntime.HandleError(err) return @@ -59,23 +66,28 @@ func (c *Controller) processNextUpdateEndpointWorkItem() bool { return false } - err := func(obj interface{}) error { + if err := func(obj interface{}) error { defer c.updateEndpointQueue.Done(obj) - var key string - var ok bool + + var ( + key string + ok bool + err error + ) + if key, ok = obj.(string); !ok { c.updateEndpointQueue.Forget(obj) utilruntime.HandleError(fmt.Errorf("expected string in workqueue but got %#v", obj)) return nil } - if err := c.handleUpdateEndpoint(key); err != nil { + + if err = c.handleUpdateEndpoint(key); err != nil { c.updateEndpointQueue.AddRateLimited(key) return fmt.Errorf("error syncing '%s': %s, requeuing", key, err.Error()) } c.updateEndpointQueue.Forget(obj) return nil - }(obj) - if err != nil { + }(obj); err != nil { utilruntime.HandleError(err) return true } @@ -112,55 +124,48 @@ func (c *Controller) handleUpdateEndpoint(key string) error { } svc := cachedService.DeepCopy() - var LbIPs []string - if vip, ok := svc.Annotations[util.SwitchLBRuleVipsAnnotation]; ok { - LbIPs = []string{vip} - } else if LbIPs = util.ServiceClusterIPs(*svc); len(LbIPs) == 0 { - return nil - } + var ( + pods []*v1.Pod + lbVips []string + vip, vpcName, subnetName string + ok bool + ignoreHealthCheck = true + ) - pods, err := c.podsLister.Pods(namespace).List(labels.Set(svc.Spec.Selector).AsSelector()) - if err != nil { - klog.Errorf("failed to get pods for service %s in namespace %s: %v", name, namespace, err) - return err - } - - var vpcName string - for _, pod := range pods { - if len(pod.Annotations) == 0 { - continue - } + if vip, ok = svc.Annotations[util.SwitchLBRuleVipsAnnotation]; ok { + lbVips = []string{vip} for _, subset := range ep.Subsets { - for _, addr := range subset.Addresses { - if addr.IP == pod.Status.PodIP { - if vpcName = pod.Annotations[util.LogicalRouterAnnotation]; vpcName != "" { - break - } + for _, address := range subset.Addresses { + // TODO: IPv6 + if util.CheckProtocol(vip) == kubeovnv1.ProtocolIPv4 && + address.TargetRef.Name != "" { + ignoreHealthCheck = false } } - if vpcName != "" { - break - } - } - if vpcName != "" { - break } + } else if lbVips = util.ServiceClusterIPs(*svc); len(lbVips) == 0 { + return nil } - if vpcName == "" { - if vpcName = svc.Annotations[util.VpcAnnotation]; vpcName == "" { - vpcName = c.config.ClusterRouter - } + if pods, err = c.podsLister.Pods(namespace).List(labels.Set(svc.Spec.Selector).AsSelector()); err != nil { + klog.Errorf("failed to get pods for service %s in namespace %s: %v", name, namespace, err) + return err } - vpc, err := c.vpcsLister.Get(vpcName) - if err != nil { + vpcName, subnetName = c.getVpcSubnetName(pods, ep, svc) + + var ( + vpc *kubeovnv1.Vpc + svcVpc string + ) + + if vpc, err = c.vpcsLister.Get(vpcName); err != nil { klog.Errorf("failed to get vpc %s of lb, %v", vpcName, err) return err } - if svcVpc := svc.Annotations[util.VpcAnnotation]; svcVpc != vpcName { + if svcVpc = svc.Annotations[util.VpcAnnotation]; svcVpc != vpcName { if svc.Annotations == nil { svc.Annotations = make(map[string]string, 1) } @@ -177,7 +182,7 @@ func (c *Controller) handleUpdateEndpoint(key string) error { tcpLb, udpLb, sctpLb, oldTCPLb, oldUDPLb, oldSctpLb = oldTCPLb, oldUDPLb, oldSctpLb, tcpLb, udpLb, sctpLb } - for _, settingIP := range LbIPs { + for _, lbVip := range lbVips { for _, port := range svc.Spec.Ports { var lb, oldLb string switch port.Protocol { @@ -189,37 +194,159 @@ func (c *Controller) handleUpdateEndpoint(key string) error { lb, oldLb = sctpLb, oldSctpLb } - vip := util.JoinHostPort(settingIP, port.Port) - backends := getServicePortBackends(ep, pods, port, settingIP) + var ( + vip, checkIP string + backends []string + ipPortMapping, externals map[string]string + ) + vip = util.JoinHostPort(lbVip, port.Port) + + if !ignoreHealthCheck { + if checkIP, err = c.getHealthCheckVip(vpcName, subnetName, lbVip); err != nil { + return err + } + + externals = map[string]string{ + util.SwitchLBRuleSubnet: subnetName, + } + } + + ipPortMapping, backends = getIPPortMappingBackend(ep, pods, port, lbVip, checkIP, ignoreHealthCheck) // for performance reason delete lb with no backends if len(backends) != 0 { - klog.V(3).Infof("update vip %s with backends %s to LB %s", vip, backends, lb) + klog.Infof("add vip endpoint %s, backends %v to LB %s", vip, backends, lb) if err = c.OVNNbClient.LoadBalancerAddVip(lb, vip, backends...); err != nil { - klog.Errorf("failed to add vip %s with backends %s to LB %s: %v", vip, backends, lb, err) + klog.Errorf("failed to add vip %s with backends %s to LB %s: %v", lbVip, backends, lb, err) return err } + if !ignoreHealthCheck && len(ipPortMapping) != 0 { + klog.Infof("add health check ip port mapping %v to LB %s", ipPortMapping, lb) + if err = c.OVNNbClient.LoadBalancerAddHealthCheck(lb, vip, ignoreHealthCheck, ipPortMapping, externals); err != nil { + klog.Errorf("failed to add health check for vip %s with ip port mapping %s to LB %s: %v", lbVip, ipPortMapping, lb, err) + return err + } + } } else { - klog.V(3).Infof("delete vip %s from LB %s", vip, lb) - if err := c.OVNNbClient.LoadBalancerDeleteVip(lb, vip); err != nil { - klog.Errorf("failed to delete vip %s from LB %s: %v", vip, lb, err) + klog.V(3).Infof("delete vip endpoint %s from LB %s", vip, lb) + if err = c.OVNNbClient.LoadBalancerDeleteVip(lb, vip, ignoreHealthCheck); err != nil { + klog.Errorf("failed to delete vip endpoint %s from LB %s: %v", vip, lb, err) return err } - klog.V(3).Infof("delete vip %s from old LB %s", vip, lb) - if err := c.OVNNbClient.LoadBalancerDeleteVip(oldLb, vip); err != nil { - klog.Errorf("failed to delete vip %s from LB %s: %v", vip, lb, err) + + klog.V(3).Infof("delete vip endpoint %s from old LB %s", vip, oldLb) + if err = c.OVNNbClient.LoadBalancerDeleteVip(oldLb, vip, ignoreHealthCheck); err != nil { + klog.Errorf("failed to delete vip %s from LB %s: %v", vip, oldLb, err) return err } } } } - return nil } -func getServicePortBackends(endpoints *v1.Endpoints, pods []*v1.Pod, servicePort v1.ServicePort, serviceIP string) []string { - backends := []string{} - protocol := util.CheckProtocol(serviceIP) +func (c *Controller) getVpcSubnetName(pods []*v1.Pod, endpoints *v1.Endpoints, service *v1.Service) (string, string) { + var ( + vpcName string + subnetName string + ) + + for _, pod := range pods { + if len(pod.Annotations) == 0 { + continue + } + + if subnetName == "" { + subnetName = pod.Annotations[util.LogicalSwitchAnnotation] + } + + LOOP: + for _, subset := range endpoints.Subsets { + for _, addr := range subset.Addresses { + if addr.IP == pod.Status.PodIP { + if vpcName == "" { + vpcName = pod.Annotations[util.LogicalRouterAnnotation] + } + + if vpcName != "" { + break LOOP + } + } + } + } + + if vpcName != "" && subnetName != "" { + break + } + } + + if vpcName == "" { + if vpcName = service.Annotations[util.VpcAnnotation]; vpcName == "" { + vpcName = c.config.ClusterRouter + } + } + + if subnetName == "" { + subnetName = util.DefaultSubnet + } + + return vpcName, subnetName +} + +func (c *Controller) getHealthCheckVip(vpcName, subnetName, lbVip string) (string, error) { + var ( + checkVip *kubeovnv1.Vip + checkIP string + err error + ) + + if checkVip, err = c.config.KubeOvnClient.KubeovnV1().Vips().Get(context.Background(), subnetName, metav1.GetOptions{}); err != nil { + if errors.IsNotFound(err) { + if checkVip, err = c.config.KubeOvnClient. + KubeovnV1(). + Vips(). + Create(context.Background(), + &kubeovnv1.Vip{ + ObjectMeta: metav1.ObjectMeta{ + Name: subnetName, + }, + Spec: kubeovnv1.VipSpec{ + Subnet: subnetName, + }, + }, + metav1.CreateOptions{}, + ); err != nil { + klog.Errorf("failed to create health check vip from vpc %s subnet %s, %v", vpcName, subnetName, err) + return checkIP, err + } + } else { + klog.Errorf("failed to get health check vip from vpc %s subnet %s, %v", vpcName, subnetName, err) + return checkIP, err + } + } + + switch util.CheckProtocol(lbVip) { + case kubeovnv1.ProtocolIPv4: + checkIP = checkVip.Status.V4ip + case kubeovnv1.ProtocolIPv6: + checkIP = checkVip.Status.V6ip + } + if checkIP == "" { + err = fmt.Errorf("failed to get health check vip from vpc %s subnet %s", vpcName, subnetName) + klog.Error(err) + return checkIP, err + } + + return checkIP, nil +} + +func getIPPortMappingBackend(endpoints *v1.Endpoints, pods []*v1.Pod, servicePort v1.ServicePort, serviceIP, checkVip string, ignoreHealthCheck bool) (map[string]string, []string) { + var ( + ipPortMapping = map[string]string{} + backends = []string{} + protocol = util.CheckProtocol(serviceIP) + ) + for _, subset := range endpoints.Subsets { var targetPort int32 for _, port := range subset.Ports { @@ -233,13 +360,16 @@ func getServicePortBackends(endpoints *v1.Endpoints, pods []*v1.Pod, servicePort } for _, address := range subset.Addresses { + if !ignoreHealthCheck && address.TargetRef.Name != "" { + ipName := fmt.Sprintf("%s.%s", address.TargetRef.Name, endpoints.Namespace) + ipPortMapping[address.IP] = fmt.Sprintf(util.HealthCheckNamedVipTemplate, ipName, checkVip) + } if address.TargetRef == nil || address.TargetRef.Kind != "Pod" { if util.CheckProtocol(address.IP) == protocol { backends = append(backends, util.JoinHostPort(address.IP, targetPort)) } continue } - var ip string for _, pod := range pods { if pod.Name == address.TargetRef.Name { @@ -265,5 +395,5 @@ func getServicePortBackends(endpoints *v1.Endpoints, pods []*v1.Pod, servicePort } } - return backends + return ipPortMapping, backends } diff --git a/pkg/controller/external-gw.go b/pkg/controller/external_gw.go similarity index 100% rename from pkg/controller/external-gw.go rename to pkg/controller/external_gw.go diff --git a/pkg/controller/gc.go b/pkg/controller/gc.go index 02cedf3ba77..fc4d37156d0 100644 --- a/pkg/controller/gc.go +++ b/pkg/controller/gc.go @@ -252,7 +252,7 @@ func (c *Controller) gcVip() error { } vips, err := c.virtualIpsLister.List(selector) if err != nil { - klog.Errorf("failed to list VIPs: %v", err) + klog.Errorf("failed to list vips: %v", err) return err } for _, vip := range vips { @@ -390,7 +390,7 @@ func (c *Controller) markAndCleanLSP() error { } func (c *Controller) gcLoadBalancer() error { - klog.Infof("start to gc loadbalancers") + klog.Infof("start to gc load balancers") if !c.config.EnableLb { // remove lb from logical switch vpcs, err := c.vpcsLister.List(labels.Everything()) @@ -435,7 +435,6 @@ func (c *Controller) gcLoadBalancer() error { return err } } - // lbs will remove from logical switch automatically when delete lbs if err = c.OVNNbClient.DeleteLoadBalancers(nil); err != nil { klog.Errorf("delete all load balancers: %v", err) @@ -449,12 +448,16 @@ func (c *Controller) gcLoadBalancer() error { klog.Errorf("failed to list svc, %v", err) return err } - tcpVips := strset.NewWithSize(len(svcs) * 2) - udpVips := strset.NewWithSize(len(svcs) * 2) - sctpVips := strset.NewWithSize(len(svcs) * 2) - tcpSessionVips := strset.NewWithSize(len(svcs) * 2) - udpSessionVips := strset.NewWithSize(len(svcs) * 2) - sctpSessionVips := strset.NewWithSize(len(svcs) * 2) + + var ( + tcpVips = strset.NewWithSize(len(svcs) * 2) + udpVips = strset.NewWithSize(len(svcs) * 2) + sctpVips = strset.NewWithSize(len(svcs) * 2) + tcpSessionVips = strset.NewWithSize(len(svcs) * 2) + udpSessionVips = strset.NewWithSize(len(svcs) * 2) + sctpSessionVips = strset.NewWithSize(len(svcs) * 2) + ) + for _, svc := range svcs { ips := util.ServiceClusterIPs(*svc) if v, ok := svc.Annotations[util.SwitchLBRuleVipsAnnotation]; ok { @@ -493,66 +496,80 @@ func (c *Controller) gcLoadBalancer() error { klog.Errorf("failed to list vpc, %v", err) return err } - var vpcLbs []string - for _, vpc := range vpcs { - tcpLb, udpLb, sctpLb := vpc.Status.TCPLoadBalancer, vpc.Status.UDPLoadBalancer, vpc.Status.SctpLoadBalancer - tcpSessLb, udpSessLb, sctpSessLb := vpc.Status.TCPSessionLoadBalancer, vpc.Status.UDPSessionLoadBalancer, vpc.Status.SctpSessionLoadBalancer - vpcLbs = append(vpcLbs, tcpLb, udpLb, sctpLb, tcpSessLb, udpSessLb, sctpSessLb) - removeVIP := func(lbName string, svcVips *strset.Set) error { - if lbName == "" { - return nil - } + var ( + removeVip func(lbName string, svcVips *strset.Set) error + vpcLbs []string + ignoreHealthCheck = true + ) - lb, err := c.OVNNbClient.GetLoadBalancer(lbName, true) - if err != nil { - klog.Errorf("get LB %s: %v", lbName, err) - return err - } - if lb == nil { - klog.Infof("load balancer %q not found", lbName) - return nil - } + removeVip = func(lbName string, svcVips *strset.Set) error { + if lbName == "" { + return nil + } - for vip := range lb.Vips { - if !svcVips.Has(vip) { - if err = c.OVNNbClient.LoadBalancerDeleteVip(lbName, vip); err != nil { - klog.Errorf("failed to delete vip %s from LB %s: %v", vip, lbName, err) - return err - } + var ( + lb *ovnnb.LoadBalancer + err error + ) + + if lb, err = c.OVNNbClient.GetLoadBalancer(lbName, true); err != nil { + klog.Errorf("get LB %s: %v", lbName, err) + return err + } + + if lb == nil { + klog.Infof("load balancer %q already deleted", lbName) + return nil + } + + for vip := range lb.Vips { + if !svcVips.Has(vip) { + if err = c.OVNNbClient.LoadBalancerDeleteVip(lbName, vip, ignoreHealthCheck); err != nil { + klog.Errorf("failed to delete vip %s from LB %s: %v", vip, lbName, err) + return err } } - return nil } + return nil + } - if err = removeVIP(tcpLb, tcpVips); err != nil { + for _, vpc := range vpcs { + var ( + tcpLb, udpLb, sctpLb = vpc.Status.TCPLoadBalancer, vpc.Status.UDPLoadBalancer, vpc.Status.SctpLoadBalancer + tcpSessLb, udpSessLb, sctpSessLb = vpc.Status.TCPSessionLoadBalancer, vpc.Status.UDPSessionLoadBalancer, vpc.Status.SctpSessionLoadBalancer + ) + + vpcLbs = append(vpcLbs, tcpLb, udpLb, sctpLb, tcpSessLb, udpSessLb, sctpSessLb) + if err = removeVip(tcpLb, tcpVips); err != nil { return err } - if err = removeVIP(tcpSessLb, tcpSessionVips); err != nil { + if err = removeVip(tcpSessLb, tcpSessionVips); err != nil { return err } - if err = removeVIP(udpLb, udpVips); err != nil { + if err = removeVip(udpLb, udpVips); err != nil { return err } - if err = removeVIP(udpSessLb, udpSessionVips); err != nil { + if err = removeVip(udpSessLb, udpSessionVips); err != nil { return err } - if err = removeVIP(sctpLb, sctpVips); err != nil { + if err = removeVip(sctpLb, sctpVips); err != nil { return err } - if err = removeVIP(sctpSessLb, sctpSessionVips); err != nil { + if err = removeVip(sctpSessLb, sctpSessionVips); err != nil { return err } } // delete lbs - if err = c.OVNNbClient.DeleteLoadBalancers(func(lb *ovnnb.LoadBalancer) bool { - return !util.ContainsString(vpcLbs, lb.Name) - }); err != nil { + if err = c.OVNNbClient.DeleteLoadBalancers( + func(lb *ovnnb.LoadBalancer) bool { + return !util.ContainsString(vpcLbs, lb.Name) + }, + ); err != nil { klog.Errorf("delete load balancers: %v", err) return err } - return nil } diff --git a/pkg/controller/init.go b/pkg/controller/init.go index 8651b56e48c..01a88d6bb48 100644 --- a/pkg/controller/init.go +++ b/pkg/controller/init.go @@ -22,29 +22,31 @@ import ( ) func (c *Controller) InitOVN() error { - if err := c.initClusterRouter(); err != nil { + var err error + + if err = c.initClusterRouter(); err != nil { klog.Errorf("init cluster router failed: %v", err) return err } if c.config.EnableLb { - if err := c.initLoadBalancer(); err != nil { + if err = c.initLoadBalancer(); err != nil { klog.Errorf("init load balancer failed: %v", err) return err } } - if err := c.initDefaultVlan(); err != nil { + if err = c.initDefaultVlan(); err != nil { klog.Errorf("init default vlan failed: %v", err) return err } - if err := c.initNodeSwitch(); err != nil { + if err = c.initNodeSwitch(); err != nil { klog.Errorf("init node switch failed: %v", err) return err } - if err := c.initDefaultLogicalSwitch(); err != nil { + if err = c.initDefaultLogicalSwitch(); err != nil { klog.Errorf("init default switch failed: %v", err) return err } @@ -199,18 +201,22 @@ func (c *Controller) initClusterRouter() error { func (c *Controller) initLB(name, protocol string, sessionAffinity bool) error { protocol = strings.ToLower(protocol) - var selectFields string + var ( + selectFields string + err error + ) + if sessionAffinity { selectFields = ovnnb.LoadBalancerSelectionFieldsIPSrc } - if err := c.OVNNbClient.CreateLoadBalancer(name, protocol, selectFields); err != nil { + if err = c.OVNNbClient.CreateLoadBalancer(name, protocol, selectFields); err != nil { klog.Errorf("create load balancer %s: %v", name, err) return err } if sessionAffinity { - if err := c.OVNNbClient.SetLoadBalancerAffinityTimeout(name, util.DefaultServiceSessionStickinessTimeout); err != nil { + if err = c.OVNNbClient.SetLoadBalancerAffinityTimeout(name, util.DefaultServiceSessionStickinessTimeout); err != nil { klog.Errorf("failed to set affinity timeout of %s load balancer %s: %v", protocol, name, err) return err } @@ -375,7 +381,7 @@ func (c *Controller) InitIPAM() error { vips, err := c.virtualIpsLister.List(labels.Everything()) if err != nil { - klog.Errorf("failed to list VIPs: %v", err) + klog.Errorf("failed to list vips: %v", err) return err } for _, vip := range vips { diff --git a/pkg/controller/ovn_dnat.go b/pkg/controller/ovn_dnat.go index 329ea12a7dd..c9833a4f3a2 100644 --- a/pkg/controller/ovn_dnat.go +++ b/pkg/controller/ovn_dnat.go @@ -21,8 +21,11 @@ import ( ) func (c *Controller) enqueueAddOvnDnatRule(obj interface{}) { - var key string - var err error + var ( + key string + err error + ) + if key, err = cache.MetaNamespaceKeyFunc(obj); err != nil { utilruntime.HandleError(err) return @@ -32,8 +35,11 @@ func (c *Controller) enqueueAddOvnDnatRule(obj interface{}) { } func (c *Controller) enqueueUpdateOvnDnatRule(oldObj, newObj interface{}) { - var key string - var err error + var ( + key string + err error + ) + if key, err = cache.MetaNamespaceKeyFunc(newObj); err != nil { utilruntime.HandleError(err) return @@ -64,8 +70,11 @@ func (c *Controller) enqueueUpdateOvnDnatRule(oldObj, newObj interface{}) { } func (c *Controller) enqueueDelOvnDnatRule(obj interface{}) { - var key string - var err error + var ( + key string + err error + ) + if key, err = cache.MetaNamespaceKeyFunc(obj); err != nil { utilruntime.HandleError(err) return @@ -455,18 +464,25 @@ func (c *Controller) handleUpdateOvnDnatRule(key string) error { } func (c *Controller) patchOvnDnatAnnotations(key, eipName string) error { - oriDnat, err := c.ovnDnatRulesLister.Get(key) - if err != nil { + var ( + oriDnat, dnat *kubeovnv1.OvnDnatRule + err error + ) + + if oriDnat, err = c.ovnDnatRulesLister.Get(key); err != nil { if k8serrors.IsNotFound(err) { return nil } klog.Error(err) return err } + dnat = oriDnat.DeepCopy() + + var ( + needUpdateAnno bool + op string + ) - dnat := oriDnat.DeepCopy() - var needUpdateAnno bool - var op string if len(dnat.Annotations) == 0 { op = "add" dnat.Annotations = map[string]string{ @@ -493,17 +509,26 @@ func (c *Controller) patchOvnDnatAnnotations(key, eipName string) error { } func (c *Controller) patchOvnDnatStatus(key, vpcName, v4Eip, podIP, podMac string, ready bool) error { - oriDnat, err := c.ovnDnatRulesLister.Get(key) - if err != nil { + var ( + oriDnat, dnat *kubeovnv1.OvnDnatRule + err error + ) + + if oriDnat, err = c.ovnDnatRulesLister.Get(key); err != nil { if k8serrors.IsNotFound(err) { return nil } klog.Error(err) return err } - dnat := oriDnat.DeepCopy() - needUpdateLabel := false - var op string + dnat = oriDnat.DeepCopy() + + var ( + needUpdateLabel = false + changed bool + op string + ) + if len(dnat.Labels) == 0 { op = "add" needUpdateLabel = true @@ -526,7 +551,6 @@ func (c *Controller) patchOvnDnatStatus(key, vpcName, v4Eip, podIP, podMac strin } } - var changed bool if dnat.Status.Ready != ready { dnat.Status.Ready = ready changed = true @@ -579,20 +603,23 @@ func (c *Controller) patchOvnDnatStatus(key, vpcName, v4Eip, podIP, podMac strin } func (c *Controller) AddDnatRule(vpcName, dnatName, externalIP, internalIP, externalPort, internalPort, protocol string) error { - externalEndpoint := net.JoinHostPort(externalIP, externalPort) - internalEndpoint := net.JoinHostPort(internalIP, internalPort) + var ( + externalEndpoint = net.JoinHostPort(externalIP, externalPort) + internalEndpoint = net.JoinHostPort(internalIP, internalPort) + err error + ) - if err := c.OVNNbClient.CreateLoadBalancer(dnatName, protocol, ""); err != nil { + if err = c.OVNNbClient.CreateLoadBalancer(dnatName, protocol, ""); err != nil { klog.Errorf("create loadBalancer %s: %v", dnatName, err) return err } - if err := c.OVNNbClient.LoadBalancerAddVip(dnatName, externalEndpoint, internalEndpoint); err != nil { + if err = c.OVNNbClient.LoadBalancerAddVip(dnatName, externalEndpoint, internalEndpoint); err != nil { klog.Errorf("add vip %s with backends %s to LB %s: %v", externalEndpoint, internalEndpoint, dnatName, err) return err } - if err := c.OVNNbClient.LogicalRouterUpdateLoadBalancers(vpcName, ovsdb.MutateOperationInsert, dnatName); err != nil { + if err = c.OVNNbClient.LogicalRouterUpdateLoadBalancers(vpcName, ovsdb.MutateOperationInsert, dnatName); err != nil { klog.Errorf("add lb %s to vpc %s: %v", dnatName, vpcName, err) return err } @@ -600,14 +627,19 @@ func (c *Controller) AddDnatRule(vpcName, dnatName, externalIP, internalIP, exte } func (c *Controller) DelDnatRule(vpcName, dnatName, externalIP, externalPort string) error { - externalEndpoint := net.JoinHostPort(externalIP, externalPort) - - if err := c.OVNNbClient.LoadBalancerDeleteVip(dnatName, externalEndpoint); err != nil { + var ( + ignoreHealthCheck = true + externalEndpoint string + err error + ) + externalEndpoint = net.JoinHostPort(externalIP, externalPort) + + if err = c.OVNNbClient.LoadBalancerDeleteVip(dnatName, externalEndpoint, ignoreHealthCheck); err != nil { klog.Errorf("delete loadBalancer vips %s: %v", externalEndpoint, err) return err } - if err := c.OVNNbClient.LogicalRouterUpdateLoadBalancers(vpcName, ovsdb.MutateOperationDelete, dnatName); err != nil { + if err = c.OVNNbClient.LogicalRouterUpdateLoadBalancers(vpcName, ovsdb.MutateOperationDelete, dnatName); err != nil { klog.Errorf("failed to remove lb %s from vpc %s: %v", dnatName, vpcName, err) return err } @@ -621,14 +653,20 @@ func (c *Controller) handleAddOvnDnatFinalizer(cachedDnat *kubeovnv1.OvnDnatRule return nil } } - newDnat := cachedDnat.DeepCopy() + + var ( + newDnat = cachedDnat.DeepCopy() + patch []byte + err error + ) + controllerutil.AddFinalizer(newDnat, finalizer) - patch, err := util.GenerateMergePatchPayload(cachedDnat, newDnat) - if err != nil { + if patch, err = util.GenerateMergePatchPayload(cachedDnat, newDnat); err != nil { klog.Errorf("failed to generate patch payload for ovn dnat '%s', %v", cachedDnat.Name, err) return err } - if _, err := c.config.KubeOvnClient.KubeovnV1().OvnDnatRules().Patch(context.Background(), cachedDnat.Name, + + if _, err = c.config.KubeOvnClient.KubeovnV1().OvnDnatRules().Patch(context.Background(), cachedDnat.Name, types.MergePatchType, patch, metav1.PatchOptions{}, ""); err != nil { if k8serrors.IsNotFound(err) { return nil @@ -643,15 +681,20 @@ func (c *Controller) handleDelOvnDnatFinalizer(cachedDnat *kubeovnv1.OvnDnatRule if len(cachedDnat.Finalizers) == 0 { return nil } - var err error - newDnat := cachedDnat.DeepCopy() + + var ( + newDnat = cachedDnat.DeepCopy() + patch []byte + err error + ) + controllerutil.RemoveFinalizer(newDnat, finalizer) - patch, err := util.GenerateMergePatchPayload(cachedDnat, newDnat) - if err != nil { + if patch, err = util.GenerateMergePatchPayload(cachedDnat, newDnat); err != nil { klog.Errorf("failed to generate patch payload for ovn dnat '%s', %v", cachedDnat.Name, err) return err } - if _, err := c.config.KubeOvnClient.KubeovnV1().OvnDnatRules().Patch(context.Background(), cachedDnat.Name, + + if _, err = c.config.KubeOvnClient.KubeovnV1().OvnDnatRules().Patch(context.Background(), cachedDnat.Name, types.MergePatchType, patch, metav1.PatchOptions{}, ""); err != nil { if k8serrors.IsNotFound(err) { return nil diff --git a/pkg/controller/ovn-ic.go b/pkg/controller/ovn_ic.go similarity index 100% rename from pkg/controller/ovn-ic.go rename to pkg/controller/ovn_ic.go diff --git a/pkg/controller/ovn-ic_test.go b/pkg/controller/ovn_ic_test.go similarity index 100% rename from pkg/controller/ovn-ic_test.go rename to pkg/controller/ovn_ic_test.go diff --git a/pkg/controller/provider-network.go b/pkg/controller/provider_network.go similarity index 100% rename from pkg/controller/provider-network.go rename to pkg/controller/provider_network.go diff --git a/pkg/controller/service.go b/pkg/controller/service.go index c41ed3fee30..89563782b3d 100644 --- a/pkg/controller/service.go +++ b/pkg/controller/service.go @@ -247,8 +247,12 @@ func (c *Controller) handleDeleteService(service *vpcService) error { return err } - vpcLbConfig := c.GenVpcLoadBalancer(service.Vpc) - var vpcLB [2]string + var ( + vpcLB [2]string + vpcLbConfig = c.GenVpcLoadBalancer(service.Vpc) + ignoreHealthCheck = true + ) + switch service.Protocol { case v1.ProtocolTCP: vpcLB = [2]string{vpcLbConfig.TCPLoadBalancer, vpcLbConfig.TCPSessLoadBalancer} @@ -259,8 +263,12 @@ func (c *Controller) handleDeleteService(service *vpcService) error { } for _, vip := range service.Vips { - var found bool - ip := parseVipAddr(vip) + var ( + ip string + found bool + ) + ip = parseVipAddr(vip) + for _, svc := range svcs { if util.ContainsString(util.ServiceClusterIPs(*svc), ip) { found = true @@ -272,7 +280,7 @@ func (c *Controller) handleDeleteService(service *vpcService) error { } for _, lb := range vpcLB { - if err := c.OVNNbClient.LoadBalancerDeleteVip(lb, vip); err != nil { + if err = c.OVNNbClient.LoadBalancerDeleteVip(lb, vip, ignoreHealthCheck); err != nil { klog.Errorf("failed to delete vip %s from LB %s: %v", vip, lb, err) return err } @@ -349,7 +357,12 @@ func (c *Controller) handleUpdateService(key string) error { } } } - needUpdateEndpointQueue := false + + var ( + needUpdateEndpointQueue = false + ignoreHealthCheck = true + ) + // for service update updateVip := func(lbName, oLbName string, svcVips []string) error { if len(lbName) == 0 { @@ -363,7 +376,7 @@ func (c *Controller) handleUpdateService(key string) error { } klog.V(3).Infof("existing vips of LB %s: %v", lbName, lb.Vips) for _, vip := range svcVips { - if err := c.OVNNbClient.LoadBalancerDeleteVip(oLbName, vip); err != nil { + if err := c.OVNNbClient.LoadBalancerDeleteVip(oLbName, vip, ignoreHealthCheck); err != nil { klog.Errorf("failed to delete vip %s from LB %s: %v", vip, oLbName, err) return err } @@ -378,7 +391,7 @@ func (c *Controller) handleUpdateService(key string) error { for vip := range lb.Vips { if ip := parseVipAddr(vip); (util.ContainsString(ips, ip) && !util.IsStringIn(vip, svcVips)) || util.ContainsString(ipsToDel, ip) { klog.Infof("remove stale vip %s from LB %s", vip, lb) - if err := c.OVNNbClient.LoadBalancerDeleteVip(lbName, vip); err != nil { + if err := c.OVNNbClient.LoadBalancerDeleteVip(lbName, vip, ignoreHealthCheck); err != nil { klog.Errorf("failed to delete vip %s from LB %s: %v", vip, lb, err) return err } @@ -398,7 +411,7 @@ func (c *Controller) handleUpdateService(key string) error { for vip := range oLb.Vips { if ip := parseVipAddr(vip); util.ContainsString(ips, ip) || util.ContainsString(ipsToDel, ip) { klog.Infof("remove stale vip %s from LB %s", vip, oLbName) - if err = c.OVNNbClient.LoadBalancerDeleteVip(oLbName, vip); err != nil { + if err = c.OVNNbClient.LoadBalancerDeleteVip(oLbName, vip, ignoreHealthCheck); err != nil { klog.Errorf("failed to delete vip %s from LB %s: %v", vip, oLbName, err) return err } diff --git a/pkg/controller/switch_lb_rule.go b/pkg/controller/switch_lb_rule.go index 24b9ca9ee8f..2f76b928466 100644 --- a/pkg/controller/switch_lb_rule.go +++ b/pkg/controller/switch_lb_rule.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "reflect" + "strconv" "strings" corev1 "k8s.io/api/core/v1" @@ -16,6 +17,7 @@ import ( "k8s.io/klog/v2" kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1" + "github.com/kubeovn/kube-ovn/pkg/ovsdb/ovnnb" "github.com/kubeovn/kube-ovn/pkg/util" ) @@ -23,6 +25,7 @@ type SlrInfo struct { Name string Namespace string IsRecreate bool + Vips []string } func generateSvcName(name string) string { @@ -30,28 +33,54 @@ func generateSvcName(name string) string { } func NewSlrInfo(slr *kubeovnv1.SwitchLBRule) *SlrInfo { + var ( + vips []string + namespace string + ) + + if namespace = slr.Spec.Namespace; namespace == "" { + namespace = "default" + } + + for _, port := range slr.Spec.Ports { + vips = append( + vips, + strings.Join( + []string{slr.Spec.Vip, strconv.Itoa(int(port.Port))}, + ":", + ), + ) + } + return &SlrInfo{ Name: slr.Name, - Namespace: slr.Spec.Namespace, + Namespace: namespace, IsRecreate: false, + Vips: vips, } } func (c *Controller) enqueueAddSwitchLBRule(obj interface{}) { - var key string - var err error + var ( + key string + err error + ) + if key, err = cache.MetaNamespaceKeyFunc(obj); err != nil { utilruntime.HandleError(err) return } + klog.Infof("enqueue add SwitchLBRule %s", key) c.addSwitchLBRuleQueue.Add(key) } func (c *Controller) enqueueUpdateSwitchLBRule(oldObj, newObj interface{}) { - oldSlr := oldObj.(*kubeovnv1.SwitchLBRule) - newSlr := newObj.(*kubeovnv1.SwitchLBRule) - info := NewSlrInfo(oldSlr) + var ( + oldSlr = oldObj.(*kubeovnv1.SwitchLBRule) + newSlr = newObj.(*kubeovnv1.SwitchLBRule) + info = NewSlrInfo(oldSlr) + ) if oldSlr.ResourceVersion == newSlr.ResourceVersion || reflect.DeepEqual(oldSlr.Spec, newSlr.Spec) { @@ -67,17 +96,18 @@ func (c *Controller) enqueueUpdateSwitchLBRule(oldObj, newObj interface{}) { } func (c *Controller) enqueueDeleteSwitchLBRule(obj interface{}) { - var key string - var err error + var ( + key string + err error + ) + if key, err = cache.MetaNamespaceKeyFunc(obj); err != nil { utilruntime.HandleError(err) return } - klog.Infof("enqueue del SwitchLBRule %s", key) - slr := obj.(*kubeovnv1.SwitchLBRule) - info := NewSlrInfo(slr) - c.delSwitchLBRuleQueue.Add(info) + klog.Infof("enqueue del SwitchLBRule %s", key) + c.delSwitchLBRuleQueue.Add(NewSlrInfo(obj.(*kubeovnv1.SwitchLBRule))) } func (c *Controller) processSwitchLBRuleWorkItem(processName string, queue workqueue.RateLimitingInterface, handler func(key *SlrInfo) error) bool { @@ -86,7 +116,7 @@ func (c *Controller) processSwitchLBRuleWorkItem(processName string, queue workq return false } - err := func(obj interface{}) error { + if err := func(obj interface{}) error { defer queue.Done(obj) key, ok := obj.(*SlrInfo) if !ok { @@ -99,8 +129,7 @@ func (c *Controller) processSwitchLBRuleWorkItem(processName string, queue workq } queue.Forget(obj) return nil - }(obj) - if err != nil { + }(obj); err != nil { utilruntime.HandleError(fmt.Errorf("process: %s. err: %v", processName, err)) queue.AddRateLimited(obj) return true @@ -202,9 +231,12 @@ func (c *Controller) handleAddOrUpdateSwitchLBRule(key string) error { } } - formatPorts := "" - newSlr := slr.DeepCopy() + var ( + formatPorts string + newSlr = slr.DeepCopy() + ) newSlr.Status.Service = fmt.Sprintf("%s/%s", svc.Namespace, svc.Name) + for _, port := range newSlr.Spec.Ports { protocol := port.Protocol if len(protocol) == 0 { @@ -225,15 +257,95 @@ func (c *Controller) handleAddOrUpdateSwitchLBRule(key string) error { func (c *Controller) handleDelSwitchLBRule(info *SlrInfo) error { klog.V(3).Infof("handleDelSwitchLBRule %s", info.Name) - name := generateSvcName(info.Name) - err := c.config.KubeClient.CoreV1().Services(info.Namespace).Delete(context.Background(), name, metav1.DeleteOptions{}) - if err != nil { + var ( + name string + lbhcs []ovnnb.LoadBalancerHealthCheck + vips map[string]struct{} + err error + ) + + name = generateSvcName(info.Name) + if err = c.config.KubeClient.CoreV1().Services(info.Namespace).Delete(context.Background(), name, metav1.DeleteOptions{}); err != nil { if k8serrors.IsNotFound(err) { return nil } - klog.Errorf("failed to delete service %s,err: %v", name, err) + klog.Errorf("failed to delete service %s, err: %v", name, err) + return err + } + + if lbhcs, err = c.OVNNbClient.ListLoadBalancerHealthChecks( + func(lbhc *ovnnb.LoadBalancerHealthCheck) bool { + return util.ContainsString(info.Vips, lbhc.Vip) + }, + ); err != nil && !k8serrors.IsNotFound(err) { + klog.Errorf("failed to list load balancer health checks matched vips %s, err: %v", info.Vips, err) + return err + } + + vips = make(map[string]struct{}) + for _, lbhc := range lbhcs { + var ( + lbs []ovnnb.LoadBalancer + vip string + ex bool + ) + + if lbs, err = c.OVNNbClient.ListLoadBalancers( + func(lb *ovnnb.LoadBalancer) bool { + return util.ContainsString(lb.HealthCheck, lbhc.UUID) + }, + ); err != nil && !k8serrors.IsNotFound(err) { + klog.Errorf("failed to list load balancer matched vips %s, err: %v", lbhc.Vip, err) + return err + } + + for _, lb := range lbs { + err = c.OVNNbClient.LoadBalancerDeleteHealthCheck(lb.Name, lbhc.UUID) + if err != nil && !k8serrors.IsNotFound(err) { + klog.Errorf("failed to delete load balancer health checks health checks %s from load balancer matched vip %s, err: %v", lbhc.Vip, lb.Name, err) + return err + } + + err = c.OVNNbClient.LoadBalancerDeleteIPPortMapping(lb.Name, lbhc.Vip) + if err != nil && !k8serrors.IsNotFound(err) { + klog.Errorf("failed to delete ip port mappings %s from load balancer matched vip %s, err: %v", lbhc.Vip, lb.Name, err) + return err + } + } + + if vip, ex = lbhc.ExternalIDs[util.SwitchLBRuleSubnet]; ex && vip != "" { + vips[vip] = struct{}{} + } + } + + if err = c.OVNNbClient.DeleteLoadBalancerHealthChecks( + func(lbhc *ovnnb.LoadBalancerHealthCheck) bool { + return util.ContainsString(info.Vips, lbhc.Vip) + }, + ); err != nil && !k8serrors.IsNotFound(err) { + klog.Errorf("delete load balancer health checks matched vip %s, err: %v", info.Vips, err) return err } + + for vip := range vips { + if lbhcs, err = c.OVNNbClient.ListLoadBalancerHealthChecks( + func(lbhc *ovnnb.LoadBalancerHealthCheck) bool { + return lbhc.ExternalIDs[util.SwitchLBRuleSubnet] == vip + }, + ); err != nil && !k8serrors.IsNotFound(err) { + klog.Errorf("failed to list load balancer, err: %v", err) + return err + } + + if len(lbhcs) == 0 { + err = c.config.KubeOvnClient.KubeovnV1().Vips().Delete(context.Background(), vip, metav1.DeleteOptions{}) + if err != nil { + klog.Errorf("failed to delete vip %s for load balancer health check, err: %v", vip, err) + return err + } + } + } + return nil } @@ -330,11 +442,14 @@ func generateEndpoints(slr *kubeovnv1.SwitchLBRule, oldEps *corev1.Endpoints) *c ) } - for _, addr := range slr.Spec.Endpoints { + for _, endpoint := range slr.Spec.Endpoints { addrs = append( addrs, corev1.EndpointAddress{ - IP: addr, + IP: endpoint, + TargetRef: &corev1.ObjectReference{ + Namespace: slr.Namespace, + }, }, ) } diff --git a/pkg/controller/vip.go b/pkg/controller/vip.go index 42128dfcea8..e5d0fa15b31 100644 --- a/pkg/controller/vip.go +++ b/pkg/controller/vip.go @@ -559,7 +559,7 @@ func (c *Controller) releaseVip(key string) error { mac = nil } if _, _, _, err = c.ipam.GetStaticAddress(key, vip.Name, vip.Status.V4ip, mac, vip.Spec.Subnet, false); err != nil { - klog.Errorf("failed to recover IPAM from VIP CR %s: %v", vip.Name, err) + klog.Errorf("failed to recover IPAM from vip CR %s: %v", vip.Name, err) } c.updateSubnetStatusQueue.Add(vip.Spec.Subnet) } diff --git a/pkg/ovs/interface.go b/pkg/ovs/interface.go index a5b8268a906..8e38930d4ed 100644 --- a/pkg/ovs/interface.go +++ b/pkg/ovs/interface.go @@ -79,7 +79,12 @@ type LogicalSwitchPort interface { type LoadBalancer interface { CreateLoadBalancer(lbName, protocol, selectFields string) error LoadBalancerAddVip(lbName, vip string, backends ...string) error - LoadBalancerDeleteVip(lbName, vip string) error + LoadBalancerDeleteVip(lbName, vip string, ignoreHealthCheck bool) error + LoadBalancerAddIPPortMapping(lbName, vip string, ipPortMappings map[string]string) error + LoadBalancerUpdateIPPortMapping(lbName, vip string, ipPortMappings map[string]string) error + LoadBalancerDeleteIPPortMapping(lbName, vip string) error + LoadBalancerAddHealthCheck(lbName, vip string, ignoreHealthCheck bool, ipPortMapping, externals map[string]string) error + LoadBalancerDeleteHealthCheck(lbName, uuid string) error SetLoadBalancerAffinityTimeout(lbName string, timeout int) error DeleteLoadBalancers(filter func(lb *ovnnb.LoadBalancer) bool) error GetLoadBalancer(lbName string, ignoreNotFound bool) (*ovnnb.LoadBalancer, error) @@ -87,6 +92,16 @@ type LoadBalancer interface { LoadBalancerExists(lbName string) (bool, error) } +type LoadBalancerHealthCheck interface { + AddLoadBalancerHealthCheck(lbName, vip string, externals map[string]string) error + CreateLoadBalancerHealthCheck(lbName, vip string, lbhc *ovnnb.LoadBalancerHealthCheck) error + DeleteLoadBalancerHealthCheck(lbName, vip string) error + DeleteLoadBalancerHealthChecks(filter func(lbhc *ovnnb.LoadBalancerHealthCheck) bool) error + GetLoadBalancerHealthCheck(lbName, vip string, ignoreNotFound bool) (*ovnnb.LoadBalancer, *ovnnb.LoadBalancerHealthCheck, error) + ListLoadBalancerHealthChecks(filter func(lbhc *ovnnb.LoadBalancerHealthCheck) bool) ([]ovnnb.LoadBalancerHealthCheck, error) + LoadBalancerHealthCheckExists(lbName, vip string) (bool, error) +} + type PortGroup interface { CreatePortGroup(pgName string, externalIDs map[string]string) error PortGroupAddPorts(pgName string, lspNames ...string) error @@ -165,6 +180,7 @@ type NbClient interface { BFD DHCPOptions LoadBalancer + LoadBalancerHealthCheck LogicalRouterPolicy LogicalRouterPort LogicalRouterStaticRoute diff --git a/pkg/ovs/ovn-nb-load_balancer.go b/pkg/ovs/ovn-nb-load_balancer.go index 120a98146ee..134b8b49306 100644 --- a/pkg/ovs/ovn-nb-load_balancer.go +++ b/pkg/ovs/ovn-nb-load_balancer.go @@ -3,6 +3,7 @@ package ovs import ( "context" "fmt" + "net" "sort" "strconv" "strings" @@ -13,22 +14,31 @@ import ( ovsclient "github.com/kubeovn/kube-ovn/pkg/ovsdb/client" "github.com/kubeovn/kube-ovn/pkg/ovsdb/ovnnb" + "github.com/kubeovn/kube-ovn/pkg/util" ) // CreateLoadBalancer create loadbalancer func (c *OVNNbClient) CreateLoadBalancer(lbName, protocol, selectFields string) error { - exist, err := c.LoadBalancerExists(lbName) - if err != nil { - klog.Error(err) + var ( + exist bool + err error + ) + + if exist, err = c.LoadBalancerExists(lbName); err != nil { + klog.Errorf("failed to get lb: %v", err) return err } - // found, ignore if exist { return nil } - lb := &ovnnb.LoadBalancer{ + var ( + ops []ovsdb.Operation + lb *ovnnb.LoadBalancer + ) + + lb = &ovnnb.LoadBalancer{ UUID: ovsclient.NamedUUID(), Name: lbName, Protocol: &protocol, @@ -38,150 +48,223 @@ func (c *OVNNbClient) CreateLoadBalancer(lbName, protocol, selectFields string) lb.SelectionFields = []string{selectFields} } - op, err := c.ovsDbClient.Create(lb) - if err != nil { + if ops, err = c.ovsDbClient.Create(lb); err != nil { return fmt.Errorf("generate operations for creating load balancer %s: %v", lbName, err) } - if err := c.Transact("lb-add", op); err != nil { + if err = c.Transact("lb-add", ops); err != nil { return fmt.Errorf("create load balancer %s: %v", lbName, err) } - return nil } // UpdateLoadBalancer update load balancer func (c *OVNNbClient) UpdateLoadBalancer(lb *ovnnb.LoadBalancer, fields ...interface{}) error { - op, err := c.ovsDbClient.Where(lb).Update(lb, fields...) - if err != nil { + var ( + ops []ovsdb.Operation + err error + ) + + if ops, err = c.ovsDbClient.Where(lb).Update(lb, fields...); err != nil { return fmt.Errorf("generate operations for updating load balancer %s: %v", lb.Name, err) } - if err = c.Transact("lb-update", op); err != nil { + if err = c.Transact("lb-update", ops); err != nil { return fmt.Errorf("update load balancer %s: %v", lb.Name, err) } - return nil } // LoadBalancerAddVips adds or updates a vip func (c *OVNNbClient) LoadBalancerAddVip(lbName, vip string, backends ...string) error { + var ( + ops []ovsdb.Operation + err error + ) + + if _, err = c.GetLoadBalancer(lbName, false); err != nil { + klog.Errorf("failed to get lb health check: %v", err) + return err + } + sort.Strings(backends) - ops, err := c.LoadBalancerOp(lbName, func(lb *ovnnb.LoadBalancer) []model.Mutation { - mutations := make([]model.Mutation, 0, 2) - value := strings.Join(backends, ",") - if len(lb.Vips) != 0 { - if lb.Vips[vip] == value { - return nil + if ops, err = c.LoadBalancerOp( + lbName, + func(lb *ovnnb.LoadBalancer) []model.Mutation { + var ( + mutations = make([]model.Mutation, 0, 2) + value = strings.Join(backends, ",") + ) + + if len(lb.Vips) != 0 { + if lb.Vips[vip] == value { + return nil + } + mutations = append( + mutations, + model.Mutation{ + Field: &lb.Vips, + Value: map[string]string{vip: lb.Vips[vip]}, + Mutator: ovsdb.MutateOperationDelete, + }, + ) } - mutations = append(mutations, model.Mutation{ - Field: &lb.Vips, - Value: map[string]string{vip: lb.Vips[vip]}, - Mutator: ovsdb.MutateOperationDelete, - }) - } - mutations = append(mutations, model.Mutation{ - Field: &lb.Vips, - Value: map[string]string{vip: value}, - Mutator: ovsdb.MutateOperationInsert, - }) - return mutations - }) - if err != nil { + mutations = append( + mutations, + model.Mutation{ + Field: &lb.Vips, + Value: map[string]string{vip: value}, + Mutator: ovsdb.MutateOperationInsert, + }, + ) + return mutations + }, + ); err != nil { return fmt.Errorf("failed to generate operations when adding vip %s with backends %v to load balancers %s: %v", vip, backends, lbName, err) } - if err = c.Transact("lb-add", ops); err != nil { - return fmt.Errorf("failed to add vip %s with backends %v to load balancers %s: %v", vip, backends, lbName, err) + + if ops != nil { + if err = c.Transact("lb-add", ops); err != nil { + return fmt.Errorf("failed to add vip %s with backends %v to load balancers %s: %v", vip, backends, lbName, err) + } } return nil } // LoadBalancerDeleteVip deletes load balancer vip -func (c *OVNNbClient) LoadBalancerDeleteVip(lbName, vip string) error { - ops, err := c.LoadBalancerOp(lbName, func(lb *ovnnb.LoadBalancer) []model.Mutation { - if len(lb.Vips) == 0 { - return nil +func (c *OVNNbClient) LoadBalancerDeleteVip(lbName, vipEndpoint string, ignoreHealthCheck bool) error { + var ( + ops []ovsdb.Operation + lb *ovnnb.LoadBalancer + lbhc *ovnnb.LoadBalancerHealthCheck + err error + ) + + lb, lbhc, err = c.GetLoadBalancerHealthCheck(lbName, vipEndpoint, true) + if err != nil { + klog.Errorf("failed to get lb health check: %v", err) + return err + } + if !ignoreHealthCheck && lbhc != nil { + klog.Infof("clean health check for lb %s with vip %s", lbName, vipEndpoint) + // delete ip port mapping + if err = c.LoadBalancerDeleteIPPortMapping(lbName, vipEndpoint); err != nil { + klog.Errorf("failed to delete lb ip port mapping: %v", err) + return err } - if _, ok := lb.Vips[vip]; !ok { - return nil + + if lbhc != nil { + if err = c.LoadBalancerDeleteHealthCheck(lbName, lbhc.UUID); err != nil { + klog.Errorf("failed to delete lb health check: %v", err) + return err + } } + } + if lb == nil || len(lb.Vips) == 0 { + return nil + } + if _, ok := lb.Vips[vipEndpoint]; !ok { + return nil + } - return []model.Mutation{{ - Field: &lb.Vips, - Value: map[string]string{vip: lb.Vips[vip]}, - Mutator: ovsdb.MutateOperationDelete, - }} - }) + ops, err = c.LoadBalancerOp( + lbName, + func(lb *ovnnb.LoadBalancer) []model.Mutation { + return []model.Mutation{ + { + Field: &lb.Vips, + Value: map[string]string{vipEndpoint: lb.Vips[vipEndpoint]}, + Mutator: ovsdb.MutateOperationDelete, + }, + } + }, + ) if err != nil { - return fmt.Errorf("failed to generate operations when deleting vip %s from load balancers %s: %v", vip, lbName, err) + return fmt.Errorf("failed to generate operations when deleting vip %s from load balancers %s: %v", vipEndpoint, lbName, err) } if len(ops) == 0 { return nil } + if err = c.Transact("lb-add", ops); err != nil { - return fmt.Errorf("failed to delete vip %s from load balancers %s: %v", vip, lbName, err) + return fmt.Errorf("failed to delete vip %s from load balancers %s: %v", vipEndpoint, lbName, err) } return nil } // SetLoadBalancerAffinityTimeout sets the LB's affinity timeout in seconds func (c *OVNNbClient) SetLoadBalancerAffinityTimeout(lbName string, timeout int) error { - lb, err := c.GetLoadBalancer(lbName, false) - if err != nil { - klog.Error(err) + var ( + options map[string]string + lb *ovnnb.LoadBalancer + value string + err error + ) + + if lb, err = c.GetLoadBalancer(lbName, false); err != nil { + klog.Errorf("failed to get lb: %v", err) return err } - value := strconv.Itoa(timeout) - if len(lb.Options) != 0 && lb.Options["affinity_timeout"] == value { + + if value = strconv.Itoa(timeout); len(lb.Options) != 0 && lb.Options["affinity_timeout"] == value { return nil } - options := make(map[string]string, len(lb.Options)+1) + options = make(map[string]string, len(lb.Options)+1) for k, v := range lb.Options { options[k] = v } options["affinity_timeout"] = value + lb.Options = options - if err := c.UpdateLoadBalancer(lb, &lb.Options); err != nil { + if err = c.UpdateLoadBalancer(lb, &lb.Options); err != nil { return fmt.Errorf("failed to set affinity timeout of lb %s to %d: %v", lbName, timeout, err) } - return nil } // DeleteLoadBalancers delete several loadbalancer once func (c *OVNNbClient) DeleteLoadBalancers(filter func(lb *ovnnb.LoadBalancer) bool) error { - op, err := c.ovsDbClient.WhereCache(func(lb *ovnnb.LoadBalancer) bool { - if filter != nil { - return filter(lb) - } - - return true - }).Delete() - if err != nil { + var ( + ops []ovsdb.Operation + err error + ) + + if ops, err = c.ovsDbClient.WhereCache( + func(lb *ovnnb.LoadBalancer) bool { + if filter != nil { + return filter(lb) + } + return true + }, + ).Delete(); err != nil { return fmt.Errorf("generate operations for delete load balancers: %v", err) } - if err := c.Transact("lb-del", op); err != nil { + if err = c.Transact("lb-del", ops); err != nil { + klog.Errorf("failed to del lbs: %v", err) return fmt.Errorf("delete load balancers : %v", err) } - return nil } // DeleteLoadBalancer delete loadbalancer func (c *OVNNbClient) DeleteLoadBalancer(lbName string) error { - op, err := c.DeleteLoadBalancerOp(lbName) + var ( + ops []ovsdb.Operation + err error + ) + + ops, err = c.DeleteLoadBalancerOp(lbName) if err != nil { - klog.Error(err) + klog.Errorf("failed to get delete lb op: %v", err) return err } - if err := c.Transact("lb-del", op); err != nil { - klog.Error(err) + if err = c.Transact("lb-del", ops); err != nil { + klog.Errorf("failed to del lb: %v", err) return fmt.Errorf("delete load balancer %s: %v", lbName, err) } - return nil } @@ -191,27 +274,33 @@ func (c *OVNNbClient) GetLoadBalancer(lbName string, ignoreNotFound bool) (*ovnn ctx, cancel := context.WithTimeout(context.Background(), c.Timeout) defer cancel() - lbList := make([]ovnnb.LoadBalancer, 0) - if err := c.ovsDbClient.WhereCache(func(lb *ovnnb.LoadBalancer) bool { - return lb.Name == lbName - }).List(ctx, &lbList); err != nil { - return nil, fmt.Errorf("list load balancer %q: %v", lbName, err) + var ( + lbList []ovnnb.LoadBalancer + err error + ) + + lbList = make([]ovnnb.LoadBalancer, 0) + if err = c.ovsDbClient.WhereCache( + func(lb *ovnnb.LoadBalancer) bool { + return lb.Name == lbName + }, + ).List(ctx, &lbList); err != nil { + return nil, fmt.Errorf("failed to list load balancer %q: %v", lbName, err) } + switch { // not found - if len(lbList) == 0 { + case len(lbList) == 0: if ignoreNotFound { return nil, nil } return nil, fmt.Errorf("not found load balancer %q", lbName) - } - - if len(lbList) > 1 { + case len(lbList) > 1: return nil, fmt.Errorf("more than one load balancer with same name %q", lbName) + default: + // #nosec G602 + return &lbList[0], nil } - - // #nosec G602 - return &lbList[0], nil } func (c *OVNNbClient) LoadBalancerExists(lbName string) (bool, error) { @@ -224,23 +313,35 @@ func (c *OVNNbClient) ListLoadBalancers(filter func(lb *ovnnb.LoadBalancer) bool ctx, cancel := context.WithTimeout(context.Background(), c.Timeout) defer cancel() - lbList := make([]ovnnb.LoadBalancer, 0) - if err := c.ovsDbClient.WhereCache(func(lb *ovnnb.LoadBalancer) bool { - if filter != nil { - return filter(lb) - } + var ( + lbList []ovnnb.LoadBalancer + err error + ) - return true - }).List(ctx, &lbList); err != nil { - return nil, fmt.Errorf("list load balancer: %v", err) - } + lbList = make([]ovnnb.LoadBalancer, 0) + if err = c.ovsDbClient.WhereCache( + func(lb *ovnnb.LoadBalancer) bool { + if filter != nil { + return filter(lb) + } + return true + }, + ).List(ctx, &lbList); err != nil { + return nil, fmt.Errorf("failed to list load balancer: %v", err) + } return lbList, nil } func (c *OVNNbClient) LoadBalancerOp(lbName string, mutationsFunc ...func(lb *ovnnb.LoadBalancer) []model.Mutation) ([]ovsdb.Operation, error) { - lb, err := c.GetLoadBalancer(lbName, false) - if err != nil { + var ( + mutations []model.Mutation + ops []ovsdb.Operation + lb *ovnnb.LoadBalancer + err error + ) + + if lb, err = c.GetLoadBalancer(lbName, false); err != nil { klog.Error(err) return nil, err } @@ -249,43 +350,238 @@ func (c *OVNNbClient) LoadBalancerOp(lbName string, mutationsFunc ...func(lb *ov return nil, nil } - mutations := make([]model.Mutation, 0, len(mutationsFunc)) + mutations = make([]model.Mutation, 0, len(mutationsFunc)) for _, f := range mutationsFunc { - if m := f(lb); len(m) != 0 { - mutations = append(mutations, m...) + if mutation := f(lb); mutation != nil { + mutations = append(mutations, mutation...) } } if len(mutations) == 0 { return nil, nil } - ops, err := c.ovsDbClient.Where(lb).Mutate(lb, mutations...) - if err != nil { + if ops, err = c.ovsDbClient.Where(lb).Mutate(lb, mutations...); err != nil { klog.Error(err) return nil, fmt.Errorf("generate operations for mutating load balancer %s: %v", lb.Name, err) } - return ops, nil } // DeleteLoadBalancerOp create operation which delete load balancer func (c *OVNNbClient) DeleteLoadBalancerOp(lbName string) ([]ovsdb.Operation, error) { - lb, err := c.GetLoadBalancer(lbName, true) - if err != nil { + var ( + ops []ovsdb.Operation + lb *ovnnb.LoadBalancer + err error + ) + + if lb, err = c.GetLoadBalancer(lbName, true); err != nil { klog.Error(err) return nil, err } - // not found, skip if lb == nil { return nil, nil } - op, err := c.Where(lb).Delete() - if err != nil { + if ops, err = c.Where(lb).Delete(); err != nil { klog.Error(err) return nil, err } + return ops, nil +} - return op, nil +// LoadBalancerAddIPPortMapping add load balancer ip port mapping +func (c *OVNNbClient) LoadBalancerAddIPPortMapping(lbName, vipEndpoint string, mappings map[string]string) error { + if len(mappings) == 0 { + return nil + } + + var ( + ops []ovsdb.Operation + err error + ) + + if ops, err = c.LoadBalancerOp( + lbName, + func(lb *ovnnb.LoadBalancer) []model.Mutation { + return []model.Mutation{ + { + Field: &lb.IPPortMappings, + Value: mappings, + Mutator: ovsdb.MutateOperationInsert, + }, + } + }, + ); err != nil { + return fmt.Errorf("failed to generate operations when adding ip port mapping with vip %v to load balancers %s: %v", vipEndpoint, lbName, err) + } + + if err = c.Transact("lb-add", ops); err != nil { + return fmt.Errorf("failed to add ip port mapping with vip %v to load balancers %s: %v", vipEndpoint, lbName, err) + } + return nil +} + +// LoadBalancerDeleteIPPortMapping delete load balancer ip port mapping +func (c *OVNNbClient) LoadBalancerDeleteIPPortMapping(lbName, vipEndpoint string) error { + var ( + ops []ovsdb.Operation + lb *ovnnb.LoadBalancer + mappings map[string]string + vip string + err error + ) + + if lb, err = c.GetLoadBalancer(lbName, true); err != nil { + klog.Errorf("failed to get lb health check: %v", err) + return err + } + + if lb == nil { + klog.Infof("lb %s already deleted", lbName) + return nil + } + if len(lb.IPPortMappings) == 0 { + klog.Infof("lb %s has no ip port mapping", lbName) + return nil + } + + if vip, _, err = net.SplitHostPort(vipEndpoint); err != nil { + err := fmt.Errorf("failed to split host port: %v", err) + klog.Error(err) + return err + } + + mappings = lb.IPPortMappings + for portIP, portMapVip := range lb.IPPortMappings { + splits := strings.Split(portMapVip, ":") + if len(splits) == 2 && splits[1] == vip { + delete(mappings, portIP) + } + } + + if ops, err = c.LoadBalancerOp( + lbName, + func(lb *ovnnb.LoadBalancer) []model.Mutation { + return []model.Mutation{ + { + Field: &lb.IPPortMappings, + Value: mappings, + Mutator: ovsdb.MutateOperationDelete, + }, + } + }, + ); err != nil { + return fmt.Errorf("failed to generate operations when deleting ip port mapping %s from load balancers %s: %v", vipEndpoint, lbName, err) + } + if len(ops) == 0 { + return nil + } + if err = c.Transact("lb-del", ops); err != nil { + return fmt.Errorf("failed to delete ip port mappings %s from load balancer %s: %v", vipEndpoint, lbName, err) + } + return nil +} + +// LoadBalancerUpdateIPPortMapping update load balancer ip port mapping +func (c *OVNNbClient) LoadBalancerUpdateIPPortMapping(lbName, vipEndpoint string, ipPortMappings map[string]string) error { + if len(ipPortMappings) != 0 { + ops, err := c.LoadBalancerOp( + lbName, + func(lb *ovnnb.LoadBalancer) []model.Mutation { + return []model.Mutation{ + { + Field: &lb.IPPortMappings, + Value: ipPortMappings, + Mutator: ovsdb.MutateOperationInsert, + }, + } + }, + ) + if err != nil { + return fmt.Errorf("failed to generate operations when adding ip port mapping with vip %v to load balancers %s: %v", vipEndpoint, lbName, err) + } + if err = c.Transact("lb-add", ops); err != nil { + return fmt.Errorf("failed to add ip port mapping with vip %v to load balancers %s: %v", vipEndpoint, lbName, err) + } + } + return nil +} + +// LoadBalancerAddHealthCheck adds health check +func (c *OVNNbClient) LoadBalancerAddHealthCheck(lbName, vipEndpoint string, ignoreHealthCheck bool, ipPortMapping, externals map[string]string) error { + klog.Infof("lb %s health check use ip port mapping %v", lbName, ipPortMapping) + if err := c.LoadBalancerUpdateIPPortMapping(lbName, vipEndpoint, ipPortMapping); err != nil { + klog.Errorf("failed to update lb ip port mapping: %v", err) + return err + } + if !ignoreHealthCheck { + klog.Infof("add health check for lb %s with vip %s and health check vip maps %v", lbName, vipEndpoint, ipPortMapping) + if err := c.AddLoadBalancerHealthCheck(lbName, vipEndpoint, externals); err != nil { + klog.Errorf("failed to create lb health check: %v", err) + return err + } + } + return nil +} + +// LoadBalancerDeleteHealthCheck delete load balancer health check +func (c *OVNNbClient) LoadBalancerDeleteHealthCheck(lbName, uuid string) error { + var ( + ops []ovsdb.Operation + lb *ovnnb.LoadBalancer + err error + ) + + if lb, err = c.GetLoadBalancer(lbName, false); err != nil { + klog.Errorf("failed to get lb: %v", err) + return err + } + + if util.ContainsString(lb.HealthCheck, uuid) { + ops, err = c.LoadBalancerOp( + lbName, + func(lb *ovnnb.LoadBalancer) []model.Mutation { + return []model.Mutation{ + { + Field: &lb.HealthCheck, + Value: []string{uuid}, + Mutator: ovsdb.MutateOperationDelete, + }, + } + }, + ) + if err != nil { + return fmt.Errorf("failed to generate operations when deleting health check %s from load balancers %s: %v", uuid, lbName, err) + } + if len(ops) == 0 { + return nil + } + if err = c.Transact("lb-del", ops); err != nil { + return fmt.Errorf("failed to delete health check %s from load balancers %s: %v", uuid, lbName, err) + } + } + + return nil +} + +// LoadBalancerUpdateHealthCheckOp create operations add to or delete health check from it +func (c *OVNNbClient) LoadBalancerUpdateHealthCheckOp(lbName string, lbhcUUIDs []string, op ovsdb.Mutator) ([]ovsdb.Operation, error) { + if len(lbhcUUIDs) == 0 { + return nil, nil + } + + return c.LoadBalancerOp( + lbName, + func(lb *ovnnb.LoadBalancer) []model.Mutation { + return []model.Mutation{ + { + Field: &lb.HealthCheck, + Value: lbhcUUIDs, + Mutator: op, + }, + } + }, + ) } diff --git a/pkg/ovs/ovn-nb-load_balancer_health_check.go b/pkg/ovs/ovn-nb-load_balancer_health_check.go new file mode 100644 index 00000000000..95fd010eae8 --- /dev/null +++ b/pkg/ovs/ovn-nb-load_balancer_health_check.go @@ -0,0 +1,289 @@ +package ovs + +import ( + "context" + "fmt" + + "golang.org/x/exp/slices" + + "github.com/ovn-org/libovsdb/model" + "github.com/ovn-org/libovsdb/ovsdb" + "k8s.io/klog/v2" + + ovsclient "github.com/kubeovn/kube-ovn/pkg/ovsdb/client" + "github.com/kubeovn/kube-ovn/pkg/ovsdb/ovnnb" +) + +func (c *OVNNbClient) AddLoadBalancerHealthCheck(lbName, vipEndpoint string, externals map[string]string) error { + lbhc, err := c.newLoadBalancerHealthCheck(lbName, vipEndpoint, externals) + if err != nil { + err := fmt.Errorf("failed to new lb health check: %v", err) + klog.Error(err) + return err + } + + return c.CreateLoadBalancerHealthCheck(lbName, vipEndpoint, lbhc) +} + +// newLoadBalancerHealthCheck return hc with basic information +func (c *OVNNbClient) newLoadBalancerHealthCheck(lbName, vipEndpoint string, externals map[string]string) (*ovnnb.LoadBalancerHealthCheck, error) { + var ( + exists bool + err error + ) + + if len(lbName) == 0 { + err = fmt.Errorf("the lb name is required") + klog.Error(err) + return nil, err + } + if len(vipEndpoint) == 0 { + err = fmt.Errorf("the vip endpoint is required") + klog.Error(err) + return nil, err + } + + if exists, err = c.LoadBalancerHealthCheckExists(lbName, vipEndpoint); err != nil { + err := fmt.Errorf("get lb health check %s: %v", vipEndpoint, err) + klog.Error(err) + return nil, err + } + + // found, ignore + if exists { + klog.Infof("already exists health check with vip %s for lb %s ", vipEndpoint, lbName) + return nil, nil + } + klog.Infof("create health check for vip endpoint %s in lb %s ", vipEndpoint, lbName) + + return &ovnnb.LoadBalancerHealthCheck{ + UUID: ovsclient.NamedUUID(), + ExternalIDs: externals, + Options: map[string]string{ + "timeout": "20", + "interval": "5", + "success_count": "3", + "failure_count": "3", + }, + Vip: vipEndpoint, + }, nil +} + +// CreateLoadBalancerHealthCheck create lb health check +func (c *OVNNbClient) CreateLoadBalancerHealthCheck(lbName, vipEndpoint string, lbhc *ovnnb.LoadBalancerHealthCheck) error { + if lbhc == nil { + return nil + } + + var ( + models = make([]model.Model, 0, 1) + lbhcUUIDs = make([]string, 0, 1) + lbHcModel = model.Model(lbhc) + ops = make([]ovsdb.Operation, 0, 2) + createLbhcOp, lbHcAddOp []ovsdb.Operation + err error + ) + + models = append(models, lbHcModel) + lbhcUUIDs = append(lbhcUUIDs, lbhc.UUID) + + if createLbhcOp, err = c.ovsDbClient.Create(models...); err != nil { + klog.Error(err) + return fmt.Errorf("generate operations for creating lbhc: %v", err) + } + ops = append(ops, createLbhcOp...) + + if lbHcAddOp, err = c.LoadBalancerUpdateHealthCheckOp(lbName, lbhcUUIDs, ovsdb.MutateOperationInsert); err != nil { + err = fmt.Errorf("generate operations for adding lbhc to lb %s: %v", lbName, err) + klog.Error(err) + return err + } + ops = append(ops, lbHcAddOp...) + + if err = c.Transact("lbhc-add", ops); err != nil { + err = fmt.Errorf("failed to create lb health check for lb %s vip %s: %v", lbName, vipEndpoint, err) + klog.Error(err) + return err + } + + return nil +} + +// UpdateLoadBalancerHealthCheck update lb +func (c *OVNNbClient) UpdateLoadBalancerHealthCheck(lbhc *ovnnb.LoadBalancerHealthCheck, fields ...interface{}) error { + var ( + op []ovsdb.Operation + err error + ) + + if op, err = c.ovsDbClient.Where(lbhc).Update(lbhc, fields...); err != nil { + return fmt.Errorf("generate operations for updating lb health check %s: %v", lbhc.Vip, err) + } + + if err = c.Transact("lbhc-update", op); err != nil { + return fmt.Errorf("update lb health check %s: %v", lbhc.Vip, err) + } + + return nil +} + +// DeleteLoadBalancerHealthChecks delete several lb health checks once +func (c *OVNNbClient) DeleteLoadBalancerHealthChecks(filter func(lb *ovnnb.LoadBalancerHealthCheck) bool) error { + var ( + op []ovsdb.Operation + err error + ) + + op, err = c.ovsDbClient.WhereCache( + func(lbhc *ovnnb.LoadBalancerHealthCheck) bool { + if filter != nil { + return filter(lbhc) + } + return true + }, + ).Delete() + + if err != nil { + return fmt.Errorf("generate operations for delete lb health checks: %v", err) + } + + if err := c.Transact("lbhc-del", op); err != nil { + return fmt.Errorf("delete lb health checks : %v", err) + } + + return nil +} + +// DeleteLoadBalancerHealthCheck delete lb health check +func (c *OVNNbClient) DeleteLoadBalancerHealthCheck(lbName, vip string) error { + var ( + op []ovsdb.Operation + err error + ) + + op, err = c.DeleteLoadBalancerHealthCheckOp(lbName, vip) + if err != nil { + klog.Errorf("failed to delete lb health check: %v", err) + return err + } + + if err = c.Transact("lbhc-del", op); err != nil { + return fmt.Errorf("delete lb %s: %v", lbName, err) + } + + return nil +} + +// GetLoadBalancerHealthCheck get lb health check by vip +func (c *OVNNbClient) GetLoadBalancerHealthCheck(lbName, vipEndpoint string, ignoreNotFound bool) (*ovnnb.LoadBalancer, *ovnnb.LoadBalancerHealthCheck, error) { + ctx, cancel := context.WithTimeout(context.Background(), c.Timeout) + defer cancel() + + var ( + lb *ovnnb.LoadBalancer + err error + ) + + if lb, err = c.GetLoadBalancer(lbName, false); err != nil { + klog.Errorf("failed to get lb %s: %v", lbName, err) + return nil, nil, err + } + + if lb.HealthCheck == nil { + if ignoreNotFound { + return lb, nil, nil + } + err = fmt.Errorf("lb %s doesn't have health check", lbName) + klog.Error(err) + return nil, nil, err + } + + healthCheckList := make([]ovnnb.LoadBalancerHealthCheck, 0) + if err = c.ovsDbClient.WhereCache( + func(healthCheck *ovnnb.LoadBalancerHealthCheck) bool { + return slices.Contains(lb.HealthCheck, healthCheck.UUID) && + healthCheck.Vip == vipEndpoint + }, + ).List(ctx, &healthCheckList); err != nil { + err = fmt.Errorf("failed to list lb health check lb health check by vip %q: %v", vipEndpoint, err) + klog.Error(err) + return nil, nil, err + } + + if len(healthCheckList) > 1 { + err = fmt.Errorf("lb %s has more than one health check with the same vip %s", lbName, vipEndpoint) + klog.Error(err) + return nil, nil, err + } + if len(healthCheckList) == 0 { + if ignoreNotFound { + return lb, nil, nil + } + err = fmt.Errorf("lb %s doesn't have health check with vip %s", lbName, vipEndpoint) + klog.Error(err) + return nil, nil, err + } + // #nosec G602 + return lb, &healthCheckList[0], nil +} + +// ListLoadBalancerHealthChecks list all lb health checks +func (c *OVNNbClient) ListLoadBalancerHealthChecks(filter func(lbhc *ovnnb.LoadBalancerHealthCheck) bool) ([]ovnnb.LoadBalancerHealthCheck, error) { + ctx, cancel := context.WithTimeout(context.Background(), c.Timeout) + defer cancel() + + var ( + lbhcList []ovnnb.LoadBalancerHealthCheck + err error + ) + lbhcList = make([]ovnnb.LoadBalancerHealthCheck, 0) + + if err = c.ovsDbClient.WhereCache( + func(lbhc *ovnnb.LoadBalancerHealthCheck) bool { + if filter != nil { + return filter(lbhc) + } + return true + }, + ).List(ctx, &lbhcList); err != nil { + return nil, fmt.Errorf("list lb health check: %v", err) + } + + return lbhcList, nil +} + +// LoadBalancerHealthCheckExists get lb health check and return the result of existence +func (c *OVNNbClient) LoadBalancerHealthCheckExists(lbName, vipEndpoint string) (bool, error) { + _, lbhc, err := c.GetLoadBalancerHealthCheck(lbName, vipEndpoint, true) + if err != nil { + klog.Errorf("failed to get lb %s health check by vip endpoint %s: %v", lbName, vipEndpoint, err) + return false, err + } + return lbhc != nil, err +} + +// DeleteLoadBalancerHealthCheckOp delete operation which delete lb health check +func (c *OVNNbClient) DeleteLoadBalancerHealthCheckOp(lbName, vip string) ([]ovsdb.Operation, error) { + var ( + lbhc *ovnnb.LoadBalancerHealthCheck + err error + ) + + _, lbhc, err = c.GetLoadBalancerHealthCheck(lbName, vip, true) + if err != nil { + klog.Errorf("failed to get lb health check: %v", err) + return nil, err + } + // not found, skip + if lbhc == nil { + return nil, nil + } + + var op []ovsdb.Operation + if op, err = c.Where(lbhc).Delete(); err != nil { + klog.Errorf("failed to generate operations for deleting lb health check: %v", err) + return nil, err + } + + return op, nil +} diff --git a/pkg/ovs/ovn-nb-load_balancer_health_check_test.go b/pkg/ovs/ovn-nb-load_balancer_health_check_test.go new file mode 100644 index 00000000000..7348e0fa7f0 --- /dev/null +++ b/pkg/ovs/ovn-nb-load_balancer_health_check_test.go @@ -0,0 +1,337 @@ +package ovs + +import ( + "fmt" + "strings" + "testing" + + "github.com/ovn-org/libovsdb/ovsdb" + "github.com/stretchr/testify/require" + + "github.com/kubeovn/kube-ovn/pkg/ovsdb/ovnnb" + "github.com/kubeovn/kube-ovn/pkg/util" +) + +func (suite *OvnClientTestSuite) testAddLoadBalancerHealthCheck() { + t := suite.T() + t.Parallel() + + var ( + ovnClient = suite.ovnClient + lbName = "test-create-lb-hc" + vip = "1.1.1.1:80" + lbhc, lbhcRepeat *ovnnb.LoadBalancerHealthCheck + err error + ) + + err = ovnClient.CreateLoadBalancer(lbName, "tcp", "ip_dst") + require.NoError(t, err) + + err = ovnClient.AddLoadBalancerHealthCheck(lbName, vip, map[string]string{}) + require.NoError(t, err) + + _, lbhc, err = ovnClient.GetLoadBalancerHealthCheck(lbName, vip, false) + require.NoError(t, err) + require.Equal(t, vip, lbhc.Vip) + require.NotEmpty(t, lbhc.UUID) + + // should no err create lbhc repeatedly + err = ovnClient.AddLoadBalancerHealthCheck(lbName, vip, map[string]string{}) + require.NoError(t, err) + + _, lbhcRepeat, err = ovnClient.GetLoadBalancerHealthCheck(lbName, vip, false) + require.NoError(t, err) + require.Equal(t, vip, lbhcRepeat.Vip) + require.NotEmpty(t, lbhcRepeat.UUID) + + require.Equal(t, lbhc.UUID, lbhcRepeat.UUID) +} + +func (suite *OvnClientTestSuite) testUpdateLoadBalancerHealthCheck() { + t := suite.T() + t.Parallel() + + var ( + ovnClient = suite.ovnClient + lbName = "test-update-lb-hc" + vip = "2.2.2.2:80" + lbhc *ovnnb.LoadBalancerHealthCheck + err error + ) + + err = ovnClient.CreateLoadBalancer(lbName, "tcp", "ip_dst") + require.NoError(t, err) + + err = ovnClient.AddLoadBalancerHealthCheck(lbName, vip, map[string]string{}) + require.NoError(t, err) + + _, lbhc, err = ovnClient.GetLoadBalancerHealthCheck(lbName, vip, false) + require.NoError(t, err) + + vip = "3.3.3.3:80" + t.Run("update vip", + func(t *testing.T) { + lbhc.Vip = vip + + err = ovnClient.UpdateLoadBalancerHealthCheck(lbhc, &lbhc.Vip) + require.NoError(t, err) + + _, lbhc, err = ovnClient.GetLoadBalancerHealthCheck(lbName, vip, false) + require.NoError(t, err) + + require.Equal(t, vip, lbhc.Vip) + }, + ) +} + +func (suite *OvnClientTestSuite) testDeleteLoadBalancerHealthCheck() { + t := suite.T() + t.Parallel() + + var ( + ovnClient = suite.ovnClient + lbName = "test-del-lb-hc" + vip = "1.1.1.11:80" + err error + ) + + err = ovnClient.CreateLoadBalancer(lbName, "tcp", "ip_dst") + require.NoError(t, err) + + err = ovnClient.AddLoadBalancerHealthCheck(lbName, vip, map[string]string{}) + require.NoError(t, err) + + err = ovnClient.DeleteLoadBalancerHealthCheck(lbName, vip) + require.NoError(t, err) + + _, _, err = ovnClient.GetLoadBalancerHealthCheck(lbName, vip, false) + require.Error(t, err) +} + +func (suite *OvnClientTestSuite) testDeleteLoadBalancerHealthChecks() { + t := suite.T() + t.Parallel() + + var ( + ovnClient = suite.ovnClient + lbNamePrefix = "test-del-lb-hcs" + vipFormat = "1.1.1.%d:80" + lbhc *ovnnb.LoadBalancerHealthCheck + vips []string + lbName, vip string + err error + ) + + for i := 0; i < 5; i++ { + lbName = fmt.Sprintf("%s-%d", lbNamePrefix, i) + vip = fmt.Sprintf(vipFormat, i+1) + + err = ovnClient.CreateLoadBalancer(lbName, "tcp", "ip_dst") + require.NoError(t, err) + + err = ovnClient.AddLoadBalancerHealthCheck(lbName, vip, map[string]string{}) + require.NoError(t, err) + + vips = append(vips, vip) + + _, lbhc, err = ovnClient.GetLoadBalancerHealthCheck(lbName, vip, false) + require.NoError(t, err) + + require.Equal(t, vip, lbhc.Vip) + } + + err = ovnClient.DeleteLoadBalancerHealthChecks( + func(lbhc *ovnnb.LoadBalancerHealthCheck) bool { + return util.ContainsString(vips, lbhc.Vip) + }, + ) + require.NoError(t, err) + + for _, ip := range vips { + _, _, err = ovnClient.GetLoadBalancerHealthCheck(lbName, ip, true) + require.NoError(t, err) + + _, _, err = ovnClient.GetLoadBalancerHealthCheck(lbName, ip, false) + require.Error(t, err) + } +} + +func (suite *OvnClientTestSuite) testGetLoadBalancerHealthCheck() { + t := suite.T() + t.Parallel() + + var ( + ovnClient = suite.ovnClient + lbName = "test-get-lb-hc" + vip = "1.1.1.22:80" + vipNonExistent = "1.1.1.33:80" + lbhc *ovnnb.LoadBalancerHealthCheck + err error + ) + + err = ovnClient.CreateLoadBalancer(lbName, "tcp", "ip_dst") + require.NoError(t, err) + + err = ovnClient.AddLoadBalancerHealthCheck(lbName, vip, map[string]string{}) + require.NoError(t, err) + + t.Run("should return no err when found load balancer health check", + func(t *testing.T) { + t.Parallel() + _, lbhc, err = ovnClient.GetLoadBalancerHealthCheck(lbName, vip, false) + require.NoError(t, err) + require.Equal(t, vip, lbhc.Vip) + require.NotEmpty(t, lbhc.UUID) + }, + ) + + t.Run("should return err when not found load balancer health check", + func(t *testing.T) { + t.Parallel() + _, _, err = ovnClient.GetLoadBalancerHealthCheck(lbName, vipNonExistent, false) + require.Error(t, err) + }, + ) + + t.Run("no err when not found load balancer health check and ignoreNotFound is true", + func(t *testing.T) { + t.Parallel() + _, _, err = ovnClient.GetLoadBalancerHealthCheck(lbName, vipNonExistent, true) + require.NoError(t, err) + }, + ) +} + +func (suite *OvnClientTestSuite) testListLoadBalancerHealthChecks() { + t := suite.T() + t.Parallel() + + var ( + ovnClient = suite.ovnClient + lbNamePrefix = "test-list-lb-hcs" + vipFormat = "1.1.1.%d:80" + vips, newVips []string + lbhcs []ovnnb.LoadBalancerHealthCheck + lbName, vip string + err error + ) + + vips = make([]string, 0, 5) + for i := 101; i <= 105; i++ { + lbName = fmt.Sprintf("%s-%d", lbNamePrefix, i) + vip = fmt.Sprintf(vipFormat, i+1) + + err = ovnClient.CreateLoadBalancer(lbName, "tcp", "ip_dst") + require.NoError(t, err) + + err = ovnClient.AddLoadBalancerHealthCheck(lbName, vip, map[string]string{}) + require.NoError(t, err) + + vips = append(vips, vip) + } + + t.Run("has no custom filter", + func(t *testing.T) { + t.Parallel() + + lbhcs, err = ovnClient.ListLoadBalancerHealthChecks(nil) + require.NoError(t, err) + require.NotEmpty(t, lbhcs) + + newVips = make([]string, 0, 5) + for _, lbhc := range lbhcs { + newVips = append(newVips, lbhc.Vip) + } + + require.Subset(t, newVips, vips) + }, + ) + + t.Run("has custom filter", + func(t *testing.T) { + t.Parallel() + + t.Run("fliter by vip", + func(t *testing.T) { + t.Parallel() + + lbhcs, err = ovnClient.ListLoadBalancerHealthChecks( + func(lbhc *ovnnb.LoadBalancerHealthCheck) bool { + return strings.Contains(lbhc.Vip, "1.1.1") + }, + ) + require.NoError(t, err) + require.NotEmpty(t, lbhcs) + + newVips = make([]string, 0, 5) + for _, lbhc := range lbhcs { + if !strings.Contains(lbhc.Vip, "1.1.1.10") { + continue + } + newVips = append(newVips, lbhc.Vip) + } + require.ElementsMatch(t, vips, newVips) + }, + ) + }, + ) +} + +func (suite *OvnClientTestSuite) testDeleteLoadBalancerHealthCheckOp() { + t := suite.T() + t.Parallel() + + var ( + ovnClient = suite.ovnClient + lbName = "test-del-lb-hc-op" + vip = "1.1.1.44:80" + vipNonExistent = "1.1.1.55:80" + lbhc *ovnnb.LoadBalancerHealthCheck + ops []ovsdb.Operation + err error + ) + + err = ovnClient.CreateLoadBalancer(lbName, "tcp", "ip_dst") + require.NoError(t, err) + + err = ovnClient.AddLoadBalancerHealthCheck(lbName, vip, map[string]string{}) + require.NoError(t, err) + + _, lbhc, err = ovnClient.GetLoadBalancerHealthCheck(lbName, vip, false) + require.NoError(t, err) + + t.Run("normal delete", + func(t *testing.T) { + t.Parallel() + + ops, err = ovnClient.DeleteLoadBalancerHealthCheckOp(lbName, vip) + require.NoError(t, err) + require.Len(t, ops, 1) + + require.Equal(t, + ovsdb.Operation{ + Op: "delete", + Table: "Load_Balancer_Health_Check", + Where: []ovsdb.Condition{ + { + Column: "_uuid", + Function: "==", + Value: ovsdb.UUID{ + GoUUID: lbhc.UUID, + }, + }, + }, + }, ops[0]) + }, + ) + + t.Run("return ops is empty when delete non-existent load balancer health check", + func(t *testing.T) { + t.Parallel() + + ops, err = ovnClient.DeleteLoadBalancerHealthCheckOp(lbName, vipNonExistent) + require.NoError(t, err) + require.Len(t, ops, 0) + }, + ) +} diff --git a/pkg/ovs/ovn-nb-load_balancer_test.go b/pkg/ovs/ovn-nb-load_balancer_test.go index e4ec1170b9a..458a65384e8 100644 --- a/pkg/ovs/ovn-nb-load_balancer_test.go +++ b/pkg/ovs/ovn-nb-load_balancer_test.go @@ -2,6 +2,7 @@ package ovs import ( "fmt" + "net" "strconv" "strings" "testing" @@ -51,8 +52,8 @@ func (suite *OvnClientTestSuite) testUpdateLoadBalancer() { t.Run("update vips", func(t *testing.T) { lb.Vips = map[string]string{ "10.96.0.1:443": "192.168.20.3:6443", - "10.107.43.237:8080": "10.244.0.15:8080,10.244.0.16:8080,10.244.0.17:8080", - "[fd00:10:96::e82f]:8080": "[fc00::af4:f]:8080,[fc00::af4:10]:8080,[fc00::af4:11]:8080", + "10.107.43.238:8080": "10.244.0.15:8080,10.244.0.16:8080,10.244.0.17:8080", + "[fd00:10:96::e83f]:8080": "[fc00::af4:f]:8080,[fc00::af4:10]:8080,[fc00::af4:11]:8080", } err := ovnClient.UpdateLoadBalancer(lb, &lb.Vips) @@ -63,8 +64,8 @@ func (suite *OvnClientTestSuite) testUpdateLoadBalancer() { require.Equal(t, map[string]string{ "10.96.0.1:443": "192.168.20.3:6443", - "10.107.43.237:8080": "10.244.0.15:8080,10.244.0.16:8080,10.244.0.17:8080", - "[fd00:10:96::e82f]:8080": "[fc00::af4:f]:8080,[fc00::af4:10]:8080,[fc00::af4:11]:8080", + "10.107.43.238:8080": "10.244.0.15:8080,10.244.0.16:8080,10.244.0.17:8080", + "[fd00:10:96::e83f]:8080": "[fc00::af4:f]:8080,[fc00::af4:10]:8080,[fc00::af4:11]:8080", }, lb.Vips) }) @@ -81,57 +82,6 @@ func (suite *OvnClientTestSuite) testUpdateLoadBalancer() { }) } -func (suite *OvnClientTestSuite) testLoadBalancerAddVip() { - t := suite.T() - t.Parallel() - - ovnClient := suite.ovnClient - lbName := "test-lb-add-vip" - - vips := map[string]string{ - "10.96.0.1:443": "192.168.20.3:6443", - "10.107.43.237:8080": "10.244.0.15:8080,10.244.0.16:8080,10.244.0.17:8080", - "[fd00:10:96::e82f]:8080": "[fc00::af4:a]:8080,[fc00::af4:b]:8080,[fc00::af4:c]:8080", - } - - err := ovnClient.CreateLoadBalancer(lbName, "tcp", "") - require.NoError(t, err) - - _, err = ovnClient.GetLoadBalancer(lbName, false) - require.NoError(t, err) - - expectedVips := make(map[string]string, len(vips)) - t.Run("add new vips to load balancer", func(t *testing.T) { - for vip, backends := range vips { - err := ovnClient.LoadBalancerAddVip(lbName, vip, strings.Split(backends, ",")...) - require.NoError(t, err) - - lb, err := ovnClient.GetLoadBalancer(lbName, false) - require.NoError(t, err) - - expectedVips[vip] = backends - require.Equal(t, lb.Vips, expectedVips) - } - }) - - vips = map[string]string{ - "10.96.0.1:443": "192.168.20.3:6443,192.168.20.4:6443", - "10.96.0.112:143": "192.168.120.3:6443,192.168.120.4:6443", - } - t.Run("add new vips to load balancer repeatedly", func(t *testing.T) { - for vip, backends := range vips { - err := ovnClient.LoadBalancerAddVip(lbName, vip, strings.Split(backends, ",")...) - require.NoError(t, err) - - lb, err := ovnClient.GetLoadBalancer(lbName, false) - require.NoError(t, err) - - expectedVips[vip] = backends - require.Equal(t, expectedVips, lb.Vips) - } - }) -} - func (suite *OvnClientTestSuite) testDeleteLoadBalancers() { t := suite.T() t.Parallel() @@ -176,43 +126,6 @@ func (suite *OvnClientTestSuite) testDeleteLoadBalancer() { require.ErrorContains(t, err, "not found load balancer") } -func (suite *OvnClientTestSuite) testLoadBalancerDeleteVip() { - t := suite.T() - t.Parallel() - - ovnClient := suite.ovnClient - lbName := "test-lb-del-vip" - - vips := map[string]string{ - "10.96.0.1:443": "192.168.20.3:6443", - "10.107.43.237:8080": "10.244.0.15:8080,10.244.0.16:8080,10.244.0.17:8080", - "[fd00:10:96::e82f]:8080": "[fc00::af4:a]:8080,[fc00::af4:b]:8080,[fc00::af4:c]:8080", - } - - err := ovnClient.CreateLoadBalancer(lbName, "tcp", "") - require.NoError(t, err) - - for vip, backends := range vips { - err = ovnClient.LoadBalancerAddVip(lbName, vip, strings.Split(backends, ",")...) - require.NoError(t, err) - } - - deletedVips := []string{ - "10.96.0.1:443", - "[fd00:10:96::e82f]:8080", - "10.96.0.100:1443", // non-existent vip - } - for _, vip := range deletedVips { - err = ovnClient.LoadBalancerDeleteVip(lbName, vip) - require.NoError(t, err) - delete(vips, vip) - } - - lb, err := ovnClient.GetLoadBalancer(lbName, false) - require.NoError(t, err) - require.Equal(t, vips, lb.Vips) -} - func (suite *OvnClientTestSuite) testGetLoadBalancer() { t := suite.T() t.Parallel() @@ -249,7 +162,7 @@ func (suite *OvnClientTestSuite) testListLoadBalancers() { t.Parallel() ovnClient := suite.ovnClient - lbNamePrefix := "test-list-lb" + lbNamePrefix := "test-list-lbs" lbNames := make([]string, 0, 3) protocol := []string{"tcp", "udp"} @@ -412,3 +325,476 @@ func (suite *OvnClientTestSuite) testSetLoadBalancerAffinityTimeout() { require.Equal(t, lb.Options["affinity_timeout"], strconv.Itoa(expectedTimeout)) }) } + +func (suite *OvnClientTestSuite) testLoadBalancerAddVip() { + t := suite.T() + t.Parallel() + + var ( + ovnClient = suite.ovnClient + lbName = "test-lb-add-vip" + vips, expectedVips map[string]string + lb *ovnnb.LoadBalancer + err error + ) + + err = ovnClient.CreateLoadBalancer(lbName, "tcp", "") + require.NoError(t, err) + + _, err = ovnClient.GetLoadBalancer(lbName, false) + require.NoError(t, err) + + vips = map[string]string{ + "10.96.0.2:443": "192.168.20.3:6443", + "10.107.43.237:8080": "10.244.0.15:8080,10.244.0.16:8080,10.244.0.17:8080", + "[fd00:10:96::e82f]:8080": "[fc00::af4:a]:8080,[fc00::af4:b]:8080,[fc00::af4:c]:8080", + } + expectedVips = make(map[string]string, len(vips)) + + t.Run("add new vips to load balancer", + func(t *testing.T) { + for vip, backends := range vips { + err = ovnClient.LoadBalancerAddVip(lbName, vip, strings.Split(backends, ",")...) + require.NoError(t, err) + + expectedVips[vip] = backends + } + + lb, err = ovnClient.GetLoadBalancer(lbName, false) + require.NoError(t, err) + + require.Equal(t, lb.Vips, expectedVips) + }, + ) + + vips = map[string]string{ + "10.96.0.2:443": "192.168.20.3:6443,192.168.20.4:6443", + "10.96.0.112:143": "192.168.120.3:6443,192.168.120.4:6443", + } + + t.Run("add new vips to load balancer repeatedly", + func(t *testing.T) { + for vip, backends := range vips { + err := ovnClient.LoadBalancerAddVip(lbName, vip, strings.Split(backends, ",")...) + require.NoError(t, err) + + expectedVips[vip] = backends + } + + lb, err = ovnClient.GetLoadBalancer(lbName, false) + require.NoError(t, err) + + require.Equal(t, lb.Vips, expectedVips) + }, + ) +} + +func (suite *OvnClientTestSuite) testLoadBalancerDeleteVip() { + t := suite.T() + t.Parallel() + + var ( + ovnClient = suite.ovnClient + lbName = "test-lb-del-vip" + vips map[string]string + deletedVips []string + lb *ovnnb.LoadBalancer + err error + ) + + err = ovnClient.CreateLoadBalancer(lbName, "tcp", "") + require.NoError(t, err) + + _, err = ovnClient.GetLoadBalancer(lbName, false) + require.NoError(t, err) + + vips = map[string]string{ + "10.96.0.3:443": "192.168.20.3:6443", + "10.107.43.239:8080": "10.244.0.15:8080,10.244.0.16:8080,10.244.0.17:8080", + "[fd00:10:96::e84f]:8080": "[fc00::af4:a]:8080,[fc00::af4:b]:8080,[fc00::af4:c]:8080", + } + ignoreHealthCheck := true + for vip, backends := range vips { + err = ovnClient.LoadBalancerAddVip(lbName, vip, strings.Split(backends, ",")...) + require.NoError(t, err) + } + + deletedVips = []string{ + "10.96.0.3:443", + "[fd00:10:96::e84f]:8080", + "10.96.0.100:1443", // non-existent vip + } + + for _, vip := range deletedVips { + err = ovnClient.LoadBalancerDeleteVip(lbName, vip, ignoreHealthCheck) + require.NoError(t, err) + delete(vips, vip) + } + + lb, err = ovnClient.GetLoadBalancer(lbName, false) + require.NoError(t, err) + require.Equal(t, vips, lb.Vips) +} + +func (suite *OvnClientTestSuite) testLoadBalancerAddIPPortMapping() { + t := suite.T() + t.Parallel() + + var ( + ovnClient = suite.ovnClient + lbName = "test-lb-add-ip-port-mapping" + vips, mappings map[string]string + lb *ovnnb.LoadBalancer + err error + ) + + err = ovnClient.CreateLoadBalancer(lbName, "tcp", "") + require.NoError(t, err) + + _, err = ovnClient.GetLoadBalancer(lbName, false) + require.NoError(t, err) + + vips = map[string]string{ + "10.96.0.4:443": "192.168.20.3:6443", + "10.107.43.240:8080": "10.244.0.15:8080,10.244.0.16:8080,10.244.0.17:8080", + "[fd00:10:96::e85f]:8080": "[fc00::af4:a]:8080,[fc00::af4:b]:8080,[fc00::af4:c]:8080", + } + t.Run("add new ip port mappings to load balancer", + func(t *testing.T) { + for vip, backends := range vips { + var ( + list []string + host string + ) + list = strings.Split(backends, ",") + mappings = make(map[string]string) + + for _, backend := range list { + host, _, err = net.SplitHostPort(backend) + require.NoError(t, err) + + mappings[host] = host + } + err = ovnClient.LoadBalancerAddVip(lbName, vip, list...) + require.NoError(t, err) + + err = ovnClient.LoadBalancerAddIPPortMapping(lbName, vip, mappings) + require.NoError(t, err) + + lb, err = ovnClient.GetLoadBalancer(lbName, false) + require.NoError(t, err) + + for _, backend := range list { + backend, _, err = net.SplitHostPort(backend) + require.NoError(t, err) + + require.Contains(t, lb.IPPortMappings, backend) + } + } + }, + ) + + vips = map[string]string{ + "10.96.0.4:443": "192.168.20.3:6443,192.168.20.4:6443", + "10.96.0.112:143": "192.168.120.3:6443,192.168.120.4:6443", + } + t.Run("add new ip port mappings to load balancer repeatedly", + func(t *testing.T) { + for vip, backends := range vips { + var ( + list []string + host string + ) + list = strings.Split(backends, ",") + mappings = make(map[string]string) + + for _, backend := range list { + host, _, err = net.SplitHostPort(backend) + require.NoError(t, err) + + mappings[host] = host + } + err = ovnClient.LoadBalancerAddVip(lbName, vip, list...) + require.NoError(t, err) + + err = ovnClient.LoadBalancerAddIPPortMapping(lbName, vip, mappings) + require.NoError(t, err) + + lb, err = ovnClient.GetLoadBalancer(lbName, false) + require.NoError(t, err) + + for _, backend := range list { + backend, _, err = net.SplitHostPort(backend) + require.NoError(t, err) + + require.Contains(t, lb.IPPortMappings, backend) + } + } + }, + ) +} + +func (suite *OvnClientTestSuite) testLoadBalancerDeleteIPPortMapping() { + t := suite.T() + t.Parallel() + + var ( + ovnClient = suite.ovnClient + lbName = "test-lb-del-ip-port-mapping" + vips, mappings map[string]string + lb *ovnnb.LoadBalancer + err error + ) + + err = ovnClient.CreateLoadBalancer(lbName, "tcp", "") + require.NoError(t, err) + + _, err = ovnClient.GetLoadBalancer(lbName, false) + require.NoError(t, err) + + vips = map[string]string{ + "10.96.0.5:443": "192.168.20.3:6443", + "10.107.43.241:8080": "10.244.0.15:8080,10.244.0.16:8080,10.244.0.17:8080", + "[fd00:10:96::e86f]:8080": "[fc00::af4:a]:8080,[fc00::af4:b]:8080,[fc00::af4:c]:8080", + } + t.Run("delete ip port mappings from load balancer", + func(t *testing.T) { + for vip, backends := range vips { + var ( + list []string + vhost, host string + ) + list = strings.Split(backends, ",") + mappings = make(map[string]string) + + for _, backend := range list { + host, _, err = net.SplitHostPort(backend) + require.NoError(t, err) + + mappings[host] = host + } + + vhost, _, err = net.SplitHostPort(vip) + require.NoError(t, err) + err = ovnClient.LoadBalancerAddVip(lbName, vhost, strings.Split(backends, ",")...) + require.NoError(t, err) + + err = ovnClient.LoadBalancerAddIPPortMapping(lbName, vhost, mappings) + require.NoError(t, err) + + lb, err = ovnClient.GetLoadBalancer(lbName, false) + require.NoError(t, err) + + for _, backend := range list { + backend, _, err = net.SplitHostPort(backend) + require.NoError(t, err) + + require.Contains(t, lb.IPPortMappings, backend) + } + + err = ovnClient.LoadBalancerDeleteIPPortMapping(lbName, vip) + require.NoError(t, err) + + lb, err = ovnClient.GetLoadBalancer(lbName, false) + require.NoError(t, err) + + for _, backend := range list { + backend, _, err = net.SplitHostPort(backend) + require.NoError(t, err) + + require.NotContains(t, lb.IPPortMappings, backend) + } + + err = ovnClient.LoadBalancerAddIPPortMapping(lbName, vhost, mappings) + require.NoError(t, err) + } + }, + ) + + vips = map[string]string{ + "10.96.0.5:443": "192.168.20.3:6443,192.168.20.4:6443", + "10.96.0.112:143": "192.168.120.3:6443,192.168.120.4:6443", + } + t.Run("delete ip port mappings from load balancer repeatedly", + func(t *testing.T) { + for vip, backends := range vips { + list := strings.Split(backends, ",") + mappings = make(map[string]string) + + err = ovnClient.LoadBalancerDeleteIPPortMapping(lbName, vip) + require.NoError(t, err) + + lb, err = ovnClient.GetLoadBalancer(lbName, false) + require.NoError(t, err) + + for _, backend := range list { + backend, _, err = net.SplitHostPort(backend) + require.NoError(t, err) + + require.NotContains(t, lb.IPPortMappings, backend) + } + } + }, + ) + + vips = map[string]string{ + "[fd00:10:96::e86f]:8080": "", + } + t.Run("delete ip port mappings from load balancer repeatedly", + func(t *testing.T) { + for vip := range vips { + err = ovnClient.LoadBalancerDeleteIPPortMapping(lbName, vip) + require.NoError(t, err) + + lb, err = ovnClient.GetLoadBalancer(lbName, false) + require.NoError(t, err) + } + }, + ) +} + +func (suite *OvnClientTestSuite) testLoadBalancerWithHealthCheck() { + t := suite.T() + t.Parallel() + + var ( + ovnClient = suite.ovnClient + lbName = "test-lb-with-health-check" + vips, mappings map[string]string + lb *ovnnb.LoadBalancer + lbhc *ovnnb.LoadBalancerHealthCheck + lbhcID, vip string + err error + ) + + err = ovnClient.CreateLoadBalancer(lbName, "tcp", "") + require.NoError(t, err) + + _, err = ovnClient.GetLoadBalancer(lbName, false) + require.NoError(t, err) + + vips = map[string]string{ + "10.96.0.6:443": "192.168.20.3:6443", + "10.107.43.242:8080": "10.244.0.15:8080,10.244.0.16:8080,10.244.0.17:8080", + "[fd00:10:96::e87f]:8080": "[fc00::af4:a]:8080,[fc00::af4:b]:8080,[fc00::af4:c]:8080", + } + t.Run("add ip port mappings from load balancer", + func(t *testing.T) { + for vip, backends := range vips { + var ( + list []string + host string + ) + list = strings.Split(backends, ",") + mappings = make(map[string]string) + + for _, backend := range list { + host, _, err = net.SplitHostPort(backend) + require.NoError(t, err) + + mappings[host] = host + } + err = ovnClient.LoadBalancerAddVip(lbName, vip, list...) + require.NoError(t, err) + + err = ovnClient.LoadBalancerAddIPPortMapping(lbName, vip, mappings) + require.NoError(t, err) + + lb, err = ovnClient.GetLoadBalancer(lbName, false) + require.NoError(t, err) + + for _, backend := range list { + backend, _, err = net.SplitHostPort(backend) + require.NoError(t, err) + + require.Contains(t, lb.IPPortMappings, backend) + } + } + }, + ) + + vips = map[string]string{ + "10.96.0.6:443": "192.168.20.4:6443", + } + t.Run("update ip port mappings from load balancer repeatedly", + func(t *testing.T) { + for vip, backends := range vips { + var ( + list []string + host string + ) + list = strings.Split(backends, ",") + mappings = make(map[string]string) + + for _, backend := range list { + host, _, err = net.SplitHostPort(backend) + require.NoError(t, err) + + mappings[host] = host + } + + err = ovnClient.LoadBalancerUpdateIPPortMapping(lbName, vip, mappings) + require.NoError(t, err) + + lb, err = ovnClient.GetLoadBalancer(lbName, false) + require.NoError(t, err) + + for _, backend := range list { + backend, _, err = net.SplitHostPort(backend) + require.NoError(t, err) + + require.Contains(t, lb.IPPortMappings, backend) + } + } + }, + ) + + vip = "10.96.0.6:443" + t.Run("add new health check to load balancer", + func(t *testing.T) { + err = ovnClient.AddLoadBalancerHealthCheck(lbName, vip, map[string]string{}) + require.NoError(t, err) + + lb, err = ovnClient.GetLoadBalancer(lbName, false) + require.NoError(t, err) + _, lbhc, err = ovnClient.GetLoadBalancerHealthCheck(lbName, vip, false) + require.NoError(t, err) + lbhcID = lbhc.UUID + require.Contains(t, lb.HealthCheck, lbhcID) + }, + ) + + t.Run("add new health check to load balancer repeatedly", + func(t *testing.T) { + err = ovnClient.AddLoadBalancerHealthCheck(lbName, vip, map[string]string{}) + require.NoError(t, err) + lb, err = ovnClient.GetLoadBalancer(lbName, false) + require.NoError(t, err) + _, lbhc, err = ovnClient.GetLoadBalancerHealthCheck(lbName, vip, false) + require.NoError(t, err) + require.Contains(t, lb.HealthCheck, lbhcID) + }, + ) + + t.Run("delete health check from load balancer", + func(t *testing.T) { + err = ovnClient.LoadBalancerDeleteHealthCheck(lbName, lbhcID) + require.NoError(t, err) + + lb, err = ovnClient.GetLoadBalancer(lbName, false) + require.NoError(t, err) + require.NotContains(t, lb.HealthCheck, lbhcID) + }, + ) + + t.Run("delete health check from load balancer repeatedly", + func(t *testing.T) { + err = ovnClient.LoadBalancerDeleteHealthCheck(lbName, lbhcID) + require.NoError(t, err) + + lb, err = ovnClient.GetLoadBalancer(lbName, false) + require.NoError(t, err) + require.NotContains(t, lb.HealthCheck, lbhcID) + }, + ) +} diff --git a/pkg/ovs/ovn-nb-nat.go b/pkg/ovs/ovn-nb-nat.go index 902fbf1ea1b..c62ce6bbc6c 100644 --- a/pkg/ovs/ovn-nb-nat.go +++ b/pkg/ovs/ovn-nb-nat.go @@ -28,6 +28,7 @@ func (c *OVNNbClient) AddNat(lrName, natType, externalIP, logicalIP, logicalMac, } }) if err != nil { + klog.Errorf("failed to new nat: %v", err) return err } diff --git a/pkg/ovs/ovn-nb-suite_test.go b/pkg/ovs/ovn-nb-suite_test.go index eb55cd5998b..ede1cf56d35 100644 --- a/pkg/ovs/ovn-nb-suite_test.go +++ b/pkg/ovs/ovn-nb-suite_test.go @@ -354,6 +354,47 @@ func (suite *OvnClientTestSuite) Test_SetLoadBalancerAffinityTimeout() { suite.testSetLoadBalancerAffinityTimeout() } +func (suite *OvnClientTestSuite) Test_LoadBalancerAddIPPortMapping() { + suite.testLoadBalancerAddIPPortMapping() +} + +func (suite *OvnClientTestSuite) Test_LoadBalancerDeleteIPPortMapping() { + suite.testLoadBalancerDeleteIPPortMapping() +} + +func (suite *OvnClientTestSuite) Test_LoadBalancerWithHealthCheck() { + suite.testLoadBalancerWithHealthCheck() +} + +/* load_balancer health check unit test */ +func (suite *OvnClientTestSuite) Test_CreateLoadBalancerHealthCheck() { + suite.testAddLoadBalancerHealthCheck() +} + +func (suite *OvnClientTestSuite) Test_UpdateLoadBalancerHealthCheck() { + suite.testUpdateLoadBalancerHealthCheck() +} + +func (suite *OvnClientTestSuite) Test_DeleteLoadBalancerHealthCheck() { + suite.testDeleteLoadBalancerHealthCheck() +} + +func (suite *OvnClientTestSuite) Test_DeleteLoadBalancerHealthChecks() { + suite.testDeleteLoadBalancerHealthChecks() +} + +func (suite *OvnClientTestSuite) Test_GetLoadBalancerHealthCheck() { + suite.testGetLoadBalancerHealthCheck() +} + +func (suite *OvnClientTestSuite) Test_ListLoadBalancerHealthChecks() { + suite.testListLoadBalancerHealthChecks() +} + +func (suite *OvnClientTestSuite) Test_DeleteLoadBalancerHealthCheckOp() { + suite.testDeleteLoadBalancerHealthCheckOp() +} + /* port_group unit test */ func (suite *OvnClientTestSuite) Test_CreatePortGroup() { suite.testCreatePortGroup() @@ -741,6 +782,7 @@ func newNbClient(addr string, timeout int) (client.Client, error) { client.WithTable(&ovnnb.DHCPOptions{}), client.WithTable(&ovnnb.GatewayChassis{}), client.WithTable(&ovnnb.LoadBalancer{}), + client.WithTable(&ovnnb.LoadBalancerHealthCheck{}), client.WithTable(&ovnnb.LogicalRouterPolicy{}), client.WithTable(&ovnnb.LogicalRouterPort{}), client.WithTable(&ovnnb.LogicalRouterStaticRoute{}), diff --git a/pkg/ovs/ovn.go b/pkg/ovs/ovn.go index ce9ed276959..0f7e826adfa 100644 --- a/pkg/ovs/ovn.go +++ b/pkg/ovs/ovn.go @@ -67,6 +67,7 @@ func NewOvnNbClient(ovnNbAddr string, ovnNbTimeout int) (*OVNNbClient, error) { client.WithTable(&ovnnb.DHCPOptions{}), client.WithTable(&ovnnb.GatewayChassis{}), client.WithTable(&ovnnb.LoadBalancer{}), + client.WithTable(&ovnnb.LoadBalancerHealthCheck{}), client.WithTable(&ovnnb.LogicalRouterPolicy{}), client.WithTable(&ovnnb.LogicalRouterPort{}), client.WithTable(&ovnnb.LogicalRouterStaticRoute{}), diff --git a/pkg/util/const.go b/pkg/util/const.go index e0d7ddbc7fa..2bfd3d3d9ab 100644 --- a/pkg/util/const.go +++ b/pkg/util/const.go @@ -47,6 +47,7 @@ const ( SwitchLBRuleVipsAnnotation = "ovn.kubernetes.io/switch_lb_vip" SwitchLBRuleVip = "switch_lb_vip" + SwitchLBRuleSubnet = "switch_lb_subnet" LogicalRouterAnnotation = "ovn.kubernetes.io/logical_router" VpcAnnotation = "ovn.kubernetes.io/vpc" @@ -273,4 +274,6 @@ const ( TProxyOutputMask = 0x90003 TProxyPreroutingMark = 0x90004 TProxyPreroutingMask = 0x90004 + + HealthCheckNamedVipTemplate = "%s:%s" // ip name, health check vip ) diff --git a/test/e2e/framework/vip.go b/test/e2e/framework/vip.go index bdd37ad5c2e..4a7751bd55e 100644 --- a/test/e2e/framework/vip.go +++ b/test/e2e/framework/vip.go @@ -86,9 +86,9 @@ func (c *VipClient) Patch(original, modified *apiv1.Vip, timeout time.Duration) } if errors.Is(err, context.DeadlineExceeded) { - Failf("timed out while retrying to patch VIP %s", original.Name) + Failf("timed out while retrying to patch vip %s", original.Name) } - Failf("error occurred while retrying to patch VIP %s: %v", original.Name, err) + Failf("error occurred while retrying to patch vip %s: %v", original.Name, err) return nil } @@ -118,7 +118,7 @@ func (c *VipClient) WaitToDisappear(name string, _, timeout time.Duration) error return vip, err })).WithTimeout(timeout).Should(gomega.BeNil()) if err != nil { - return fmt.Errorf("expected VIP %s to not be found: %w", name, err) + return fmt.Errorf("expected vip %s to not be found: %w", name, err) } return nil }