Skip to content

Commit

Permalink
feat: Added NIC type and modified setRoutes function to support Windo…
Browse files Browse the repository at this point in the history 
…ws (#2679)

* Added NIC type and modified setRoutes function to support Windows

* Removed deprecated nic types
  • Loading branch information
@bohuini
bohuini committed Jun 7, 2024
1 parent 7e8b45f commit 13f7037
Show file tree
Hide file tree
Showing 10 changed files with 406 additions and 164 deletions.
2 changes: 1 addition & 1 deletion cni/network/invoker_cns.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -167,7 +167,7 @@ func (invoker *CNSIPAMInvoker) Add(addConfig IPAMAddConfig) (IPAMAddResult, erro
key := invoker.getInterfaceInfoKey(info.nicType, info.macAddress)
switch info.nicType {
case cns.DelegatedVMNIC:
// only handling single v4 PodIPInfo for DelegatedVMNICs at the moment, will have to update once v6 gets added
// only handling single v4 PodIPInfo for Frontend NICs at the moment, will have to update once v6 gets added
if !info.skipDefaultRoutes {
numInterfacesWithDefaultRoutes++
}
Expand Down
2 changes: 1 addition & 1 deletion cni/network/invoker_cns_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1444,7 +1444,7 @@ func Test_getInterfaceInfoKey(t *testing.T) {
require.Equal(string(cns.InfraNIC), inv.getInterfaceInfoKey(cns.InfraNIC, dummyMAC))
require.Equal(dummyMAC, inv.getInterfaceInfoKey(cns.DelegatedVMNIC, dummyMAC))
require.Equal("", inv.getInterfaceInfoKey(cns.DelegatedVMNIC, ""))
require.Equal(string(cns.BackendNIC), inv.getInterfaceInfoKey(cns.BackendNIC, dummyMAC))
require.Equal(string(cns.NodeNetworkInterfaceBackendNIC), inv.getInterfaceInfoKey(cns.NodeNetworkInterfaceBackendNIC, dummyMAC))
}

func TestCNSIPAMInvoker_Add_SwiftV2(t *testing.T) {
Expand Down
19 changes: 13 additions & 6 deletions cns/NetworkContainerContract.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -78,10 +78,18 @@ type NICType string
// NIC Types
const (
InfraNIC NICType = "InfraNIC"
// Delegated VM NICs are projected from VM to container network namespace
DelegatedVMNIC NICType = "DelegatedVMNIC"
// BackendNIC NICs are used for infiniband nics on a VM
// DelegatedVMNIC are projected from VM to container network namespace
DelegatedVMNIC NICType = "FrontendNIC"
// BackendNIC are used for infiniband NICs on a VM
BackendNIC NICType = "BackendNIC"
// NodeNetworkInterfaceAccelnetFrontendNIC is a type of front-end nic that offers accelerated networking performance
NodeNetworkInterfaceAccelnetFrontendNIC NICType = "FrontendNIC_Accelnet"

// TODO: These two const are currently unused due to version compatibility with DNC. DelegatedVMNIC and NodeNetworkInterfaceBackendNIC should be renamed to align with the naming convention with DNC
// NodeNetworkInterfaceFrontendNIC is the new name for DelegatedVMNIC
NodeNetworkInterfaceFrontendNIC NICType = "FrontendNIC"
// NodeNetworkInterfaceBackendNIC is the new name for BackendNIC
NodeNetworkInterfaceBackendNIC NICType = "BackendNIC"
)

// ChannelMode :- CNS channel modes
Expand Down Expand Up @@ -465,9 +473,8 @@ type PodIpInfo struct {
PodIPConfig IPSubnet
NetworkContainerPrimaryIPConfig IPConfiguration
HostPrimaryIPInfo HostIPInfo
// NICType defines whether NIC is InfraNIC or DelegatedVMNIC or BackendNIC
NICType NICType
InterfaceName string
NICType NICType
InterfaceName string
// MacAddress of interface
MacAddress string
// SkipDefaultRoutes is true if default routes should not be added on interface
Expand Down
208 changes: 67 additions & 141 deletions cns/middlewares/k8sSwiftV2.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,6 @@ package middlewares
import (
"context"
"fmt"
"net/netip"

"github.com/Azure/azure-container-networking/cns"
"github.com/Azure/azure-container-networking/cns/configuration"
"github.com/Azure/azure-container-networking/cns/logger"
Expand Down Expand Up @@ -39,17 +37,17 @@ var _ cns.IPConfigsHandlerMiddleware = (*K8sSWIFTv2Middleware)(nil)

// IPConfigsRequestHandlerWrapper is the middleware function for handling SWIFT v2 IP configs requests for AKS-SWIFT. This function wrapped the default SWIFT request
// and release IP configs handlers.
func (m *K8sSWIFTv2Middleware) IPConfigsRequestHandlerWrapper(defaultHandler, failureHandler cns.IPConfigsHandlerFunc) cns.IPConfigsHandlerFunc {
func (k *K8sSWIFTv2Middleware) IPConfigsRequestHandlerWrapper(defaultHandler, failureHandler cns.IPConfigsHandlerFunc) cns.IPConfigsHandlerFunc {
return func(ctx context.Context, req cns.IPConfigsRequest) (*cns.IPConfigsResponse, error) {
podInfo, respCode, message := m.validateIPConfigsRequest(ctx, &req)
podInfo, respCode, message := k.validateIPConfigsRequest(ctx, &req)

if respCode != types.Success {
return &cns.IPConfigsResponse{
Response: cns.Response{
ReturnCode: respCode,
Message: message,
},
}, errors.New("failed to validate ip configs request")
}, errors.New("failed to validate IP configs request")
}
ipConfigsResp, err := defaultHandler(ctx, req)
// If the pod is not v2, return the response from the handler
Expand All @@ -69,7 +67,7 @@ func (m *K8sSWIFTv2Middleware) IPConfigsRequestHandlerWrapper(defaultHandler, fa
if err != nil {
return ipConfigsResp, err
}
SWIFTv2PodIPInfo, err := m.getIPConfig(ctx, podInfo)
SWIFTv2PodIPInfos, err := k.getIPConfig(ctx, podInfo)
if err != nil {
return &cns.IPConfigsResponse{
Response: cns.Response{
Expand All @@ -79,11 +77,11 @@ func (m *K8sSWIFTv2Middleware) IPConfigsRequestHandlerWrapper(defaultHandler, fa
PodIPInfo: []cns.PodIpInfo{},
}, errors.Wrapf(err, "failed to get SWIFTv2 IP config : %v", req)
}
ipConfigsResp.PodIPInfo = append(ipConfigsResp.PodIPInfo, SWIFTv2PodIPInfo)
ipConfigsResp.PodIPInfo = append(ipConfigsResp.PodIPInfo, SWIFTv2PodIPInfos...)
// Set routes for the pod
for i := range ipConfigsResp.PodIPInfo {
ipInfo := &ipConfigsResp.PodIPInfo[i]
err = m.setRoutes(ipInfo)
err = k.setRoutes(ipInfo)
if err != nil {
return &cns.IPConfigsResponse{
Response: cns.Response{
Expand All @@ -100,7 +98,7 @@ func (m *K8sSWIFTv2Middleware) IPConfigsRequestHandlerWrapper(defaultHandler, fa

// validateIPConfigsRequest validates if pod is multitenant by checking the pod labels, used in SWIFT V2 AKS scenario.
// nolint
func (m *K8sSWIFTv2Middleware) validateIPConfigsRequest(ctx context.Context, req *cns.IPConfigsRequest) (podInfo cns.PodInfo, respCode types.ResponseCode, message string) {
func (k *K8sSWIFTv2Middleware) validateIPConfigsRequest(ctx context.Context, req *cns.IPConfigsRequest) (podInfo cns.PodInfo, respCode types.ResponseCode, message string) {
// Retrieve the pod from the cluster
podInfo, err := cns.UnmarshalPodInfo(req.OrchestratorContext)
if err != nil {
Expand All @@ -110,7 +108,7 @@ func (m *K8sSWIFTv2Middleware) validateIPConfigsRequest(ctx context.Context, req
logger.Printf("[SWIFTv2Middleware] validate ipconfigs request for pod %s", podInfo.Name())
podNamespacedName := k8stypes.NamespacedName{Namespace: podInfo.Namespace(), Name: podInfo.Name()}
pod := v1.Pod{}
if err := m.Cli.Get(ctx, podNamespacedName, &pod); err != nil {
if err := k.Cli.Get(ctx, podNamespacedName, &pod); err != nil {
errBuf := errors.Wrapf(err, "failed to get pod %+v", podNamespacedName)
return nil, types.UnexpectedError, errBuf.Error()
}
Expand All @@ -121,11 +119,11 @@ func (m *K8sSWIFTv2Middleware) validateIPConfigsRequest(ctx context.Context, req
// Check if the MTPNC CRD exists for the pod, if not, return error
mtpnc := v1alpha1.MultitenantPodNetworkConfig{}
mtpncNamespacedName := k8stypes.NamespacedName{Namespace: podInfo.Namespace(), Name: podInfo.Name()}
if err := m.Cli.Get(ctx, mtpncNamespacedName, &mtpnc); err != nil {
if err := k.Cli.Get(ctx, mtpncNamespacedName, &mtpnc); err != nil {
return nil, types.UnexpectedError, fmt.Errorf("failed to get pod's mtpnc from cache : %w", err).Error()
}
// Check if the MTPNC CRD is ready. If one of the fields is empty, return error
if mtpnc.Status.PrimaryIP == "" || mtpnc.Status.MacAddress == "" || mtpnc.Status.NCID == "" || mtpnc.Status.GatewayIP == "" {
if !mtpnc.IsReady() {
return nil, types.UnexpectedError, errMTPNCNotReady.Error()
}
}
Expand All @@ -135,150 +133,78 @@ func (m *K8sSWIFTv2Middleware) validateIPConfigsRequest(ctx context.Context, req
}

// getIPConfig returns the pod's SWIFT V2 IP configuration.
func (m *K8sSWIFTv2Middleware) getIPConfig(ctx context.Context, podInfo cns.PodInfo) (cns.PodIpInfo, error) {
func (k *K8sSWIFTv2Middleware) getIPConfig(ctx context.Context, podInfo cns.PodInfo) ([]cns.PodIpInfo, error) {
// Check if the MTPNC CRD exists for the pod, if not, return error
mtpnc := v1alpha1.MultitenantPodNetworkConfig{}
mtpncNamespacedName := k8stypes.NamespacedName{Namespace: podInfo.Namespace(), Name: podInfo.Name()}
if err := m.Cli.Get(ctx, mtpncNamespacedName, &mtpnc); err != nil {
return cns.PodIpInfo{}, errors.Wrapf(err, "failed to get pod's mtpnc from cache")
if err := k.Cli.Get(ctx, mtpncNamespacedName, &mtpnc); err != nil {
return nil, errors.Wrapf(err, "failed to get pod's mtpnc from cache")
}

// Check if the MTPNC CRD is ready. If one of the fields is empty, return error
if mtpnc.Status.PrimaryIP == "" || mtpnc.Status.MacAddress == "" || mtpnc.Status.NCID == "" || mtpnc.Status.GatewayIP == "" {
return cns.PodIpInfo{}, errMTPNCNotReady
if !mtpnc.IsReady() {
return nil, errMTPNCNotReady
}
logger.Printf("[SWIFTv2Middleware] mtpnc for pod %s is : %+v", podInfo.Name(), mtpnc)

// Parse MTPNC primaryIP to get the IP address and prefix length
p, err := netip.ParsePrefix(mtpnc.Status.PrimaryIP)
if err != nil {
return cns.PodIpInfo{}, errors.Wrapf(err, "failed to parse mtpnc primaryIP %s", mtpnc.Status.PrimaryIP)
}
// Get the IP address and prefix length
ip := p.Addr()
prefixSize := p.Bits()
if prefixSize != prefixLength {
return cns.PodIpInfo{}, errors.Wrapf(errInvalidMTPNCPrefixLength, "mtpnc primaryIP prefix length is %d", prefixSize)
}
podIPInfo := cns.PodIpInfo{
PodIPConfig: cns.IPSubnet{
IPAddress: ip.String(),
PrefixLength: uint8(prefixSize),
},
MacAddress: mtpnc.Status.MacAddress,
NICType: cns.DelegatedVMNIC,
SkipDefaultRoutes: false,
// InterfaceName is empty for DelegatedVMNIC
}

return podIPInfo, nil
}

// setRoutes sets the routes for podIPInfo used in SWIFT V2 scenario.
func (m *K8sSWIFTv2Middleware) setRoutes(podIPInfo *cns.PodIpInfo) error {
logger.Printf("[SWIFTv2Middleware] set routes for pod with nic type : %s", podIPInfo.NICType)
podIPInfo.Routes = []cns.Route{}
switch podIPInfo.NICType {
case cns.DelegatedVMNIC:
virtualGWRoute := cns.Route{
IPAddress: fmt.Sprintf("%s/%d", virtualGW, prefixLength),
}
// default route via SWIFT v2 interface
route := cns.Route{
IPAddress: "0.0.0.0/0",
GatewayIPAddress: virtualGW,
}
podIPInfo.Routes = []cns.Route{virtualGWRoute, route}
case cns.InfraNIC:
// Get and parse infraVNETCIDRs from env
infraVNETCIDRs, err := configuration.InfraVNETCIDRs()
if err != nil {
return errors.Wrapf(err, "failed to get infraVNETCIDRs from env")
}
infraVNETCIDRsv4, infraVNETCIDRsv6, err := utils.ParseCIDRs(infraVNETCIDRs)
if err != nil {
return errors.Wrapf(err, "failed to parse infraVNETCIDRs")
}

// Get and parse podCIDRs from env
podCIDRs, err := configuration.PodCIDRs()
if err != nil {
return errors.Wrapf(err, "failed to get podCIDRs from env")
}
podCIDRsV4, podCIDRv6, err := utils.ParseCIDRs(podCIDRs)
if err != nil {
return errors.Wrapf(err, "failed to parse podCIDRs")
}
var podIPInfos []cns.PodIpInfo

// Get and parse serviceCIDRs from env
serviceCIDRs, err := configuration.ServiceCIDRs()
if len(mtpnc.Status.InterfaceInfos) == 0 {
// Use fields from mtpnc.Status if InterfaceInfos is empty
ip, prefixSize, err := utils.ParseIPAndPrefix(mtpnc.Status.PrimaryIP)
if err != nil {
return errors.Wrapf(err, "failed to get serviceCIDRs from env")
}
serviceCIDRsV4, serviceCIDRsV6, err := utils.ParseCIDRs(serviceCIDRs)
if err != nil {
return errors.Wrapf(err, "failed to parse serviceCIDRs")
}
// Check if the podIPInfo is IPv4 or IPv6
ip, err := netip.ParseAddr(podIPInfo.PodIPConfig.IPAddress)
if err != nil {
return errors.Wrapf(err, "failed to parse podIPConfig IP address %s", podIPInfo.PodIPConfig.IPAddress)
}
if ip.Is4() {
// routes for IPv4 podCIDR traffic
for _, podCIDRv4 := range podCIDRsV4 {
podCIDRv4Route := cns.Route{
IPAddress: podCIDRv4,
GatewayIPAddress: overlayGatewayv4,
}
podIPInfo.Routes = append(podIPInfo.Routes, podCIDRv4Route)
}
// route for IPv4 serviceCIDR traffic
for _, serviceCIDRv4 := range serviceCIDRsV4 {
serviceCIDRv4Route := cns.Route{
IPAddress: serviceCIDRv4,
GatewayIPAddress: overlayGatewayv4,
}
podIPInfo.Routes = append(podIPInfo.Routes, serviceCIDRv4Route)
}
// route for IPv4 infraVNETCIDR traffic
for _, infraVNETCIDRv4 := range infraVNETCIDRsv4 {
infraVNETCIDRv4Route := cns.Route{
IPAddress: infraVNETCIDRv4,
GatewayIPAddress: overlayGatewayv4,
}
podIPInfo.Routes = append(podIPInfo.Routes, infraVNETCIDRv4Route)
return nil, errors.Wrap(err, "failed to parse mtpnc primary IP and prefix")
}
if prefixSize != prefixLength {
return nil, errors.Wrapf(errInvalidMTPNCPrefixLength, "mtpnc primaryIP prefix length is %d", prefixSize)
}

podIPInfos = append(podIPInfos, cns.PodIpInfo{
PodIPConfig: cns.IPSubnet{
IPAddress: ip,
PrefixLength: uint8(prefixSize),
},
MacAddress: mtpnc.Status.MacAddress,
NICType: cns.DelegatedVMNIC,
SkipDefaultRoutes: false,
// InterfaceName is empty for DelegatedVMNIC
})
} else {
// Use InterfaceInfos if not empty
podIPInfos = make([]cns.PodIpInfo, len(mtpnc.Status.InterfaceInfos))
for i, interfaceInfo := range mtpnc.Status.InterfaceInfos {
// Parse MTPNC primaryIP to get the IP address and prefix length
ip, prefixSize, err := utils.ParseIPAndPrefix(interfaceInfo.PrimaryIP)
if err != nil {
return nil, errors.Wrap(err, "failed to parse mtpnc primary IP and prefix")
}
} else {
// routes for IPv6 podCIDR traffic
for _, podCIDRv6 := range podCIDRv6 {
podCIDRv6Route := cns.Route{
IPAddress: podCIDRv6,
GatewayIPAddress: overlayGatewayV6,
}
podIPInfo.Routes = append(podIPInfo.Routes, podCIDRv6Route)
if prefixSize != prefixLength {
return nil, errors.Wrapf(errInvalidMTPNCPrefixLength, "mtpnc primaryIP prefix length is %d", prefixSize)
}
// route for IPv6 serviceCIDR traffic
for _, serviceCIDRv6 := range serviceCIDRsV6 {
serviceCIDRv6Route := cns.Route{
IPAddress: serviceCIDRv6,
GatewayIPAddress: overlayGatewayV6,
}
podIPInfo.Routes = append(podIPInfo.Routes, serviceCIDRv6Route)

var nicType cns.NICType
switch {
case interfaceInfo.DeviceType == v1alpha1.DeviceTypeVnetNIC && !interfaceInfo.AccelnetEnabled:
nicType = cns.DelegatedVMNIC
case interfaceInfo.DeviceType == v1alpha1.DeviceTypeVnetNIC && interfaceInfo.AccelnetEnabled:
nicType = cns.NodeNetworkInterfaceAccelnetFrontendNIC
case interfaceInfo.DeviceType == v1alpha1.DeviceTypeInfiniBandNIC:
nicType = cns.NodeNetworkInterfaceBackendNIC
default:
nicType = cns.DelegatedVMNIC
}
// route for IPv6 infraVNETCIDR traffic
for _, infraVNETCIDRv6 := range infraVNETCIDRsv6 {
infraVNETCIDRv6Route := cns.Route{
IPAddress: infraVNETCIDRv6,
GatewayIPAddress: overlayGatewayV6,
}
podIPInfo.Routes = append(podIPInfo.Routes, infraVNETCIDRv6Route)

podIPInfos[i] = cns.PodIpInfo{
PodIPConfig: cns.IPSubnet{
IPAddress: ip,
PrefixLength: uint8(prefixSize),
},
MacAddress: interfaceInfo.MacAddress,
NICType: nicType,
SkipDefaultRoutes: false,
}
}
podIPInfo.SkipDefaultRoutes = true
case cns.BackendNIC:
default:
return errInvalidSWIFTv2NICType
}
return nil

return podIPInfos, nil
}
Loading

0 comments on commit 13f7037

Please sign in to comment.