Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: Added NIC type and modified setRoutes function to support Windows #2679

Merged
merged 3 commits into from
Jun 8, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion cni/network/invoker_cns.go
Original file line number Diff line number Diff line change
Expand Up @@ -167,7 +167,7 @@ func (invoker *CNSIPAMInvoker) Add(addConfig IPAMAddConfig) (IPAMAddResult, erro
key := invoker.getInterfaceInfoKey(info.nicType, info.macAddress)
switch info.nicType {
case cns.DelegatedVMNIC:
// only handling single v4 PodIPInfo for DelegatedVMNICs at the moment, will have to update once v6 gets added
// only handling single v4 PodIPInfo for Frontend NICs at the moment, will have to update once v6 gets added
if !info.skipDefaultRoutes {
numInterfacesWithDefaultRoutes++
}
Expand Down
2 changes: 1 addition & 1 deletion cni/network/invoker_cns_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -1444,7 +1444,7 @@ func Test_getInterfaceInfoKey(t *testing.T) {
require.Equal(string(cns.InfraNIC), inv.getInterfaceInfoKey(cns.InfraNIC, dummyMAC))
require.Equal(dummyMAC, inv.getInterfaceInfoKey(cns.DelegatedVMNIC, dummyMAC))
require.Equal("", inv.getInterfaceInfoKey(cns.DelegatedVMNIC, ""))
require.Equal(string(cns.BackendNIC), inv.getInterfaceInfoKey(cns.BackendNIC, dummyMAC))
require.Equal(string(cns.NodeNetworkInterfaceBackendNIC), inv.getInterfaceInfoKey(cns.NodeNetworkInterfaceBackendNIC, dummyMAC))
}

func TestCNSIPAMInvoker_Add_SwiftV2(t *testing.T) {
Expand Down
19 changes: 13 additions & 6 deletions cns/NetworkContainerContract.go
Original file line number Diff line number Diff line change
Expand Up @@ -78,10 +78,18 @@ type NICType string
// NIC Types
const (
InfraNIC NICType = "InfraNIC"
// Delegated VM NICs are projected from VM to container network namespace
DelegatedVMNIC NICType = "DelegatedVMNIC"
kmurudi marked this conversation as resolved.
Show resolved Hide resolved
// BackendNIC NICs are used for infiniband nics on a VM
// DelegatedVMNIC are projected from VM to container network namespace
DelegatedVMNIC NICType = "FrontendNIC"
kmurudi marked this conversation as resolved.
Show resolved Hide resolved
// BackendNIC are used for infiniband NICs on a VM
BackendNIC NICType = "BackendNIC"
kmurudi marked this conversation as resolved.
Show resolved Hide resolved
// NodeNetworkInterfaceAccelnetFrontendNIC is a type of front-end nic that offers accelerated networking performance
NodeNetworkInterfaceAccelnetFrontendNIC NICType = "FrontendNIC_Accelnet"

// TODO: These two const are currently unused due to version compatibility with DNC. DelegatedVMNIC and NodeNetworkInterfaceBackendNIC should be renamed to align with the naming convention with DNC
// NodeNetworkInterfaceFrontendNIC is the new name for DelegatedVMNIC
NodeNetworkInterfaceFrontendNIC NICType = "FrontendNIC"
// NodeNetworkInterfaceBackendNIC is the new name for BackendNIC
NodeNetworkInterfaceBackendNIC NICType = "BackendNIC"
)

// ChannelMode :- CNS channel modes
Expand Down Expand Up @@ -465,9 +473,8 @@ type PodIpInfo struct {
PodIPConfig IPSubnet
NetworkContainerPrimaryIPConfig IPConfiguration
HostPrimaryIPInfo HostIPInfo
// NICType defines whether NIC is InfraNIC or DelegatedVMNIC or BackendNIC
NICType NICType
InterfaceName string
NICType NICType
InterfaceName string
// MacAddress of interface
MacAddress string
// SkipDefaultRoutes is true if default routes should not be added on interface
Expand Down
208 changes: 67 additions & 141 deletions cns/middlewares/k8sSwiftV2.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,6 @@ package middlewares
import (
"context"
"fmt"
"net/netip"

"github.com/Azure/azure-container-networking/cns"
"github.com/Azure/azure-container-networking/cns/configuration"
"github.com/Azure/azure-container-networking/cns/logger"
Expand Down Expand Up @@ -39,17 +37,17 @@ var _ cns.IPConfigsHandlerMiddleware = (*K8sSWIFTv2Middleware)(nil)

// IPConfigsRequestHandlerWrapper is the middleware function for handling SWIFT v2 IP configs requests for AKS-SWIFT. This function wrapped the default SWIFT request
// and release IP configs handlers.
func (m *K8sSWIFTv2Middleware) IPConfigsRequestHandlerWrapper(defaultHandler, failureHandler cns.IPConfigsHandlerFunc) cns.IPConfigsHandlerFunc {
func (k *K8sSWIFTv2Middleware) IPConfigsRequestHandlerWrapper(defaultHandler, failureHandler cns.IPConfigsHandlerFunc) cns.IPConfigsHandlerFunc {
return func(ctx context.Context, req cns.IPConfigsRequest) (*cns.IPConfigsResponse, error) {
podInfo, respCode, message := m.validateIPConfigsRequest(ctx, &req)
podInfo, respCode, message := k.validateIPConfigsRequest(ctx, &req)

if respCode != types.Success {
return &cns.IPConfigsResponse{
Response: cns.Response{
ReturnCode: respCode,
Message: message,
},
}, errors.New("failed to validate ip configs request")
}, errors.New("failed to validate IP configs request")
}
ipConfigsResp, err := defaultHandler(ctx, req)
// If the pod is not v2, return the response from the handler
Expand All @@ -69,7 +67,7 @@ func (m *K8sSWIFTv2Middleware) IPConfigsRequestHandlerWrapper(defaultHandler, fa
if err != nil {
return ipConfigsResp, err
}
SWIFTv2PodIPInfo, err := m.getIPConfig(ctx, podInfo)
SWIFTv2PodIPInfos, err := k.getIPConfig(ctx, podInfo)
if err != nil {
return &cns.IPConfigsResponse{
Response: cns.Response{
Expand All @@ -79,11 +77,11 @@ func (m *K8sSWIFTv2Middleware) IPConfigsRequestHandlerWrapper(defaultHandler, fa
PodIPInfo: []cns.PodIpInfo{},
}, errors.Wrapf(err, "failed to get SWIFTv2 IP config : %v", req)
}
ipConfigsResp.PodIPInfo = append(ipConfigsResp.PodIPInfo, SWIFTv2PodIPInfo)
ipConfigsResp.PodIPInfo = append(ipConfigsResp.PodIPInfo, SWIFTv2PodIPInfos...)
// Set routes for the pod
for i := range ipConfigsResp.PodIPInfo {
ipInfo := &ipConfigsResp.PodIPInfo[i]
err = m.setRoutes(ipInfo)
err = k.setRoutes(ipInfo)
if err != nil {
return &cns.IPConfigsResponse{
Response: cns.Response{
Expand All @@ -100,7 +98,7 @@ func (m *K8sSWIFTv2Middleware) IPConfigsRequestHandlerWrapper(defaultHandler, fa

// validateIPConfigsRequest validates if pod is multitenant by checking the pod labels, used in SWIFT V2 AKS scenario.
// nolint
func (m *K8sSWIFTv2Middleware) validateIPConfigsRequest(ctx context.Context, req *cns.IPConfigsRequest) (podInfo cns.PodInfo, respCode types.ResponseCode, message string) {
func (k *K8sSWIFTv2Middleware) validateIPConfigsRequest(ctx context.Context, req *cns.IPConfigsRequest) (podInfo cns.PodInfo, respCode types.ResponseCode, message string) {
// Retrieve the pod from the cluster
podInfo, err := cns.UnmarshalPodInfo(req.OrchestratorContext)
if err != nil {
Expand All @@ -110,7 +108,7 @@ func (m *K8sSWIFTv2Middleware) validateIPConfigsRequest(ctx context.Context, req
logger.Printf("[SWIFTv2Middleware] validate ipconfigs request for pod %s", podInfo.Name())
podNamespacedName := k8stypes.NamespacedName{Namespace: podInfo.Namespace(), Name: podInfo.Name()}
pod := v1.Pod{}
if err := m.Cli.Get(ctx, podNamespacedName, &pod); err != nil {
if err := k.Cli.Get(ctx, podNamespacedName, &pod); err != nil {
errBuf := errors.Wrapf(err, "failed to get pod %+v", podNamespacedName)
return nil, types.UnexpectedError, errBuf.Error()
}
Expand All @@ -121,11 +119,11 @@ func (m *K8sSWIFTv2Middleware) validateIPConfigsRequest(ctx context.Context, req
// Check if the MTPNC CRD exists for the pod, if not, return error
mtpnc := v1alpha1.MultitenantPodNetworkConfig{}
mtpncNamespacedName := k8stypes.NamespacedName{Namespace: podInfo.Namespace(), Name: podInfo.Name()}
if err := m.Cli.Get(ctx, mtpncNamespacedName, &mtpnc); err != nil {
if err := k.Cli.Get(ctx, mtpncNamespacedName, &mtpnc); err != nil {
return nil, types.UnexpectedError, fmt.Errorf("failed to get pod's mtpnc from cache : %w", err).Error()
}
// Check if the MTPNC CRD is ready. If one of the fields is empty, return error
if mtpnc.Status.PrimaryIP == "" || mtpnc.Status.MacAddress == "" || mtpnc.Status.NCID == "" || mtpnc.Status.GatewayIP == "" {
if !mtpnc.IsReady() {
return nil, types.UnexpectedError, errMTPNCNotReady.Error()
}
}
Expand All @@ -135,150 +133,78 @@ func (m *K8sSWIFTv2Middleware) validateIPConfigsRequest(ctx context.Context, req
}

// getIPConfig returns the pod's SWIFT V2 IP configuration.
func (m *K8sSWIFTv2Middleware) getIPConfig(ctx context.Context, podInfo cns.PodInfo) (cns.PodIpInfo, error) {
func (k *K8sSWIFTv2Middleware) getIPConfig(ctx context.Context, podInfo cns.PodInfo) ([]cns.PodIpInfo, error) {
// Check if the MTPNC CRD exists for the pod, if not, return error
mtpnc := v1alpha1.MultitenantPodNetworkConfig{}
mtpncNamespacedName := k8stypes.NamespacedName{Namespace: podInfo.Namespace(), Name: podInfo.Name()}
if err := m.Cli.Get(ctx, mtpncNamespacedName, &mtpnc); err != nil {
return cns.PodIpInfo{}, errors.Wrapf(err, "failed to get pod's mtpnc from cache")
if err := k.Cli.Get(ctx, mtpncNamespacedName, &mtpnc); err != nil {
return nil, errors.Wrapf(err, "failed to get pod's mtpnc from cache")
}

// Check if the MTPNC CRD is ready. If one of the fields is empty, return error
if mtpnc.Status.PrimaryIP == "" || mtpnc.Status.MacAddress == "" || mtpnc.Status.NCID == "" || mtpnc.Status.GatewayIP == "" {
return cns.PodIpInfo{}, errMTPNCNotReady
if !mtpnc.IsReady() {
return nil, errMTPNCNotReady
}
logger.Printf("[SWIFTv2Middleware] mtpnc for pod %s is : %+v", podInfo.Name(), mtpnc)

// Parse MTPNC primaryIP to get the IP address and prefix length
p, err := netip.ParsePrefix(mtpnc.Status.PrimaryIP)
if err != nil {
return cns.PodIpInfo{}, errors.Wrapf(err, "failed to parse mtpnc primaryIP %s", mtpnc.Status.PrimaryIP)
}
// Get the IP address and prefix length
ip := p.Addr()
prefixSize := p.Bits()
if prefixSize != prefixLength {
return cns.PodIpInfo{}, errors.Wrapf(errInvalidMTPNCPrefixLength, "mtpnc primaryIP prefix length is %d", prefixSize)
}
podIPInfo := cns.PodIpInfo{
PodIPConfig: cns.IPSubnet{
IPAddress: ip.String(),
PrefixLength: uint8(prefixSize),
},
MacAddress: mtpnc.Status.MacAddress,
NICType: cns.DelegatedVMNIC,
SkipDefaultRoutes: false,
// InterfaceName is empty for DelegatedVMNIC
}

return podIPInfo, nil
}

// setRoutes sets the routes for podIPInfo used in SWIFT V2 scenario.
func (m *K8sSWIFTv2Middleware) setRoutes(podIPInfo *cns.PodIpInfo) error {
logger.Printf("[SWIFTv2Middleware] set routes for pod with nic type : %s", podIPInfo.NICType)
podIPInfo.Routes = []cns.Route{}
switch podIPInfo.NICType {
case cns.DelegatedVMNIC:
virtualGWRoute := cns.Route{
IPAddress: fmt.Sprintf("%s/%d", virtualGW, prefixLength),
}
// default route via SWIFT v2 interface
route := cns.Route{
IPAddress: "0.0.0.0/0",
GatewayIPAddress: virtualGW,
}
podIPInfo.Routes = []cns.Route{virtualGWRoute, route}
case cns.InfraNIC:
// Get and parse infraVNETCIDRs from env
infraVNETCIDRs, err := configuration.InfraVNETCIDRs()
if err != nil {
return errors.Wrapf(err, "failed to get infraVNETCIDRs from env")
}
infraVNETCIDRsv4, infraVNETCIDRsv6, err := utils.ParseCIDRs(infraVNETCIDRs)
if err != nil {
return errors.Wrapf(err, "failed to parse infraVNETCIDRs")
}

// Get and parse podCIDRs from env
podCIDRs, err := configuration.PodCIDRs()
if err != nil {
return errors.Wrapf(err, "failed to get podCIDRs from env")
}
podCIDRsV4, podCIDRv6, err := utils.ParseCIDRs(podCIDRs)
if err != nil {
return errors.Wrapf(err, "failed to parse podCIDRs")
}
var podIPInfos []cns.PodIpInfo

// Get and parse serviceCIDRs from env
serviceCIDRs, err := configuration.ServiceCIDRs()
if len(mtpnc.Status.InterfaceInfos) == 0 {
// Use fields from mtpnc.Status if InterfaceInfos is empty
ip, prefixSize, err := utils.ParseIPAndPrefix(mtpnc.Status.PrimaryIP)
if err != nil {
return errors.Wrapf(err, "failed to get serviceCIDRs from env")
}
serviceCIDRsV4, serviceCIDRsV6, err := utils.ParseCIDRs(serviceCIDRs)
if err != nil {
return errors.Wrapf(err, "failed to parse serviceCIDRs")
}
// Check if the podIPInfo is IPv4 or IPv6
ip, err := netip.ParseAddr(podIPInfo.PodIPConfig.IPAddress)
if err != nil {
return errors.Wrapf(err, "failed to parse podIPConfig IP address %s", podIPInfo.PodIPConfig.IPAddress)
}
if ip.Is4() {
// routes for IPv4 podCIDR traffic
for _, podCIDRv4 := range podCIDRsV4 {
podCIDRv4Route := cns.Route{
IPAddress: podCIDRv4,
GatewayIPAddress: overlayGatewayv4,
}
podIPInfo.Routes = append(podIPInfo.Routes, podCIDRv4Route)
}
// route for IPv4 serviceCIDR traffic
for _, serviceCIDRv4 := range serviceCIDRsV4 {
serviceCIDRv4Route := cns.Route{
IPAddress: serviceCIDRv4,
GatewayIPAddress: overlayGatewayv4,
}
podIPInfo.Routes = append(podIPInfo.Routes, serviceCIDRv4Route)
}
// route for IPv4 infraVNETCIDR traffic
for _, infraVNETCIDRv4 := range infraVNETCIDRsv4 {
infraVNETCIDRv4Route := cns.Route{
IPAddress: infraVNETCIDRv4,
GatewayIPAddress: overlayGatewayv4,
}
podIPInfo.Routes = append(podIPInfo.Routes, infraVNETCIDRv4Route)
return nil, errors.Wrap(err, "failed to parse mtpnc primary IP and prefix")
}
if prefixSize != prefixLength {
return nil, errors.Wrapf(errInvalidMTPNCPrefixLength, "mtpnc primaryIP prefix length is %d", prefixSize)
}

podIPInfos = append(podIPInfos, cns.PodIpInfo{
PodIPConfig: cns.IPSubnet{
IPAddress: ip,
PrefixLength: uint8(prefixSize),
},
MacAddress: mtpnc.Status.MacAddress,
kmurudi marked this conversation as resolved.
Show resolved Hide resolved
NICType: cns.DelegatedVMNIC,
SkipDefaultRoutes: false,
// InterfaceName is empty for DelegatedVMNIC
paulyufan2 marked this conversation as resolved.
Show resolved Hide resolved
})
} else {
nddq marked this conversation as resolved.
Show resolved Hide resolved
// Use InterfaceInfos if not empty
podIPInfos = make([]cns.PodIpInfo, len(mtpnc.Status.InterfaceInfos))
for i, interfaceInfo := range mtpnc.Status.InterfaceInfos {
nddq marked this conversation as resolved.
Show resolved Hide resolved
// Parse MTPNC primaryIP to get the IP address and prefix length
ip, prefixSize, err := utils.ParseIPAndPrefix(interfaceInfo.PrimaryIP)
if err != nil {
return nil, errors.Wrap(err, "failed to parse mtpnc primary IP and prefix")
}
} else {
// routes for IPv6 podCIDR traffic
for _, podCIDRv6 := range podCIDRv6 {
podCIDRv6Route := cns.Route{
IPAddress: podCIDRv6,
GatewayIPAddress: overlayGatewayV6,
}
podIPInfo.Routes = append(podIPInfo.Routes, podCIDRv6Route)
if prefixSize != prefixLength {
return nil, errors.Wrapf(errInvalidMTPNCPrefixLength, "mtpnc primaryIP prefix length is %d", prefixSize)
}
// route for IPv6 serviceCIDR traffic
for _, serviceCIDRv6 := range serviceCIDRsV6 {
serviceCIDRv6Route := cns.Route{
IPAddress: serviceCIDRv6,
GatewayIPAddress: overlayGatewayV6,
}
podIPInfo.Routes = append(podIPInfo.Routes, serviceCIDRv6Route)

var nicType cns.NICType
switch {
case interfaceInfo.DeviceType == v1alpha1.DeviceTypeVnetNIC && !interfaceInfo.AccelnetEnabled:
nicType = cns.DelegatedVMNIC
case interfaceInfo.DeviceType == v1alpha1.DeviceTypeVnetNIC && interfaceInfo.AccelnetEnabled:
nicType = cns.NodeNetworkInterfaceAccelnetFrontendNIC
case interfaceInfo.DeviceType == v1alpha1.DeviceTypeInfiniBandNIC:
nicType = cns.NodeNetworkInterfaceBackendNIC
default:
nicType = cns.DelegatedVMNIC
}
// route for IPv6 infraVNETCIDR traffic
for _, infraVNETCIDRv6 := range infraVNETCIDRsv6 {
infraVNETCIDRv6Route := cns.Route{
IPAddress: infraVNETCIDRv6,
GatewayIPAddress: overlayGatewayV6,
}
podIPInfo.Routes = append(podIPInfo.Routes, infraVNETCIDRv6Route)

podIPInfos[i] = cns.PodIpInfo{
PodIPConfig: cns.IPSubnet{
IPAddress: ip,
PrefixLength: uint8(prefixSize),
},
MacAddress: interfaceInfo.MacAddress,
NICType: nicType,
SkipDefaultRoutes: false,
}
}
podIPInfo.SkipDefaultRoutes = true
case cns.BackendNIC:
default:
return errInvalidSWIFTv2NICType
}
return nil

return podIPInfos, nil
}
Loading
Loading