Skip to content

Commit

Permalink
Added NIC type and modified setRoutes function
Browse files Browse the repository at this point in the history
  • Loading branch information
bohuini committed Apr 17, 2024
1 parent 1f01781 commit aabae63
Show file tree
Hide file tree
Showing 6 changed files with 137 additions and 127 deletions.
11 changes: 6 additions & 5 deletions cns/NetworkContainerContract.go
Original file line number Diff line number Diff line change
Expand Up @@ -80,8 +80,10 @@ const (
InfraNIC NICType = "InfraNIC"
// Delegated VM NICs are projected from VM to container network namespace
DelegatedVMNIC NICType = "DelegatedVMNIC"
// BackendNIC NICs are used for infiniband nics on a VM
BackendNIC NICType = "BackendNIC"
// NodeNetworkInterfaceBackendNIC are used for infiniband NICs on a VM
NodeNetworkInterfaceBackendNIC NICType = "BackendNIC"
// NodeNetworkInterfaceAccelnetFrontendNIC is a type of front-end nic that offers accelerated networking performance
NodeNetworkInterfaceAccelnetFrontendNIC NICType = "FrontendNIC_Accelnet"
)

// ChannelMode :- CNS channel modes
Expand Down Expand Up @@ -465,9 +467,8 @@ type PodIpInfo struct {
PodIPConfig IPSubnet
NetworkContainerPrimaryIPConfig IPConfiguration
HostPrimaryIPInfo HostIPInfo
// NICType defines whether NIC is InfraNIC or DelegatedVMNIC or BackendNIC
NICType NICType
InterfaceName string
NICType NICType
InterfaceName string
// MacAddress of interface
MacAddress string
// SkipDefaultRoutes is true if default routes should not be added on interface
Expand Down
134 changes: 12 additions & 122 deletions cns/middlewares/k8sSwiftV2.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@ import (
"github.com/Azure/azure-container-networking/cns"
"github.com/Azure/azure-container-networking/cns/configuration"
"github.com/Azure/azure-container-networking/cns/logger"
"github.com/Azure/azure-container-networking/cns/middlewares/utils"
"github.com/Azure/azure-container-networking/cns/types"
"github.com/Azure/azure-container-networking/crd/multitenancy/api/v1alpha1"
"github.com/pkg/errors"
Expand Down Expand Up @@ -39,9 +38,9 @@ var _ cns.IPConfigsHandlerMiddleware = (*K8sSWIFTv2Middleware)(nil)

// IPConfigsRequestHandlerWrapper is the middleware function for handling SWIFT v2 IP configs requests for AKS-SWIFT. This function wrapped the default SWIFT request
// and release IP configs handlers.
func (m *K8sSWIFTv2Middleware) IPConfigsRequestHandlerWrapper(defaultHandler, failureHandler cns.IPConfigsHandlerFunc) cns.IPConfigsHandlerFunc {
func (k *K8sSWIFTv2Middleware) IPConfigsRequestHandlerWrapper(defaultHandler, failureHandler cns.IPConfigsHandlerFunc) cns.IPConfigsHandlerFunc {
return func(ctx context.Context, req cns.IPConfigsRequest) (*cns.IPConfigsResponse, error) {
podInfo, respCode, message := m.validateIPConfigsRequest(ctx, &req)
podInfo, respCode, message := k.validateIPConfigsRequest(ctx, &req)

if respCode != types.Success {
return &cns.IPConfigsResponse{
Expand Down Expand Up @@ -69,7 +68,7 @@ func (m *K8sSWIFTv2Middleware) IPConfigsRequestHandlerWrapper(defaultHandler, fa
if err != nil {
return ipConfigsResp, err
}
SWIFTv2PodIPInfo, err := m.getIPConfig(ctx, podInfo)
SWIFTv2PodIPInfo, err := k.getIPConfig(ctx, podInfo)
if err != nil {
return &cns.IPConfigsResponse{
Response: cns.Response{
Expand All @@ -83,7 +82,7 @@ func (m *K8sSWIFTv2Middleware) IPConfigsRequestHandlerWrapper(defaultHandler, fa
// Set routes for the pod
for i := range ipConfigsResp.PodIPInfo {
ipInfo := &ipConfigsResp.PodIPInfo[i]
err = m.setRoutes(ipInfo)
err = k.setRoutes(ipInfo)
if err != nil {
return &cns.IPConfigsResponse{
Response: cns.Response{
Expand All @@ -100,7 +99,7 @@ func (m *K8sSWIFTv2Middleware) IPConfigsRequestHandlerWrapper(defaultHandler, fa

// validateIPConfigsRequest validates if pod is multitenant by checking the pod labels, used in SWIFT V2 AKS scenario.
// nolint
func (m *K8sSWIFTv2Middleware) validateIPConfigsRequest(ctx context.Context, req *cns.IPConfigsRequest) (podInfo cns.PodInfo, respCode types.ResponseCode, message string) {
func (k *K8sSWIFTv2Middleware) validateIPConfigsRequest(ctx context.Context, req *cns.IPConfigsRequest) (podInfo cns.PodInfo, respCode types.ResponseCode, message string) {
// Retrieve the pod from the cluster
podInfo, err := cns.UnmarshalPodInfo(req.OrchestratorContext)
if err != nil {
Expand All @@ -110,7 +109,7 @@ func (m *K8sSWIFTv2Middleware) validateIPConfigsRequest(ctx context.Context, req
logger.Printf("[SWIFTv2Middleware] validate ipconfigs request for pod %s", podInfo.Name())
podNamespacedName := k8stypes.NamespacedName{Namespace: podInfo.Namespace(), Name: podInfo.Name()}
pod := v1.Pod{}
if err := m.Cli.Get(ctx, podNamespacedName, &pod); err != nil {
if err := k.Cli.Get(ctx, podNamespacedName, &pod); err != nil {
errBuf := errors.Wrapf(err, "failed to get pod %+v", podNamespacedName)
return nil, types.UnexpectedError, errBuf.Error()
}
Expand All @@ -121,11 +120,11 @@ func (m *K8sSWIFTv2Middleware) validateIPConfigsRequest(ctx context.Context, req
// Check if the MTPNC CRD exists for the pod, if not, return error
mtpnc := v1alpha1.MultitenantPodNetworkConfig{}
mtpncNamespacedName := k8stypes.NamespacedName{Namespace: podInfo.Namespace(), Name: podInfo.Name()}
if err := m.Cli.Get(ctx, mtpncNamespacedName, &mtpnc); err != nil {
if err := k.Cli.Get(ctx, mtpncNamespacedName, &mtpnc); err != nil {
return nil, types.UnexpectedError, fmt.Errorf("failed to get pod's mtpnc from cache : %w", err).Error()
}
// Check if the MTPNC CRD is ready. If one of the fields is empty, return error
if mtpnc.Status.PrimaryIP == "" || mtpnc.Status.MacAddress == "" || mtpnc.Status.NCID == "" || mtpnc.Status.GatewayIP == "" {
if !mtpnc.IsReady() {
return nil, types.UnexpectedError, errMTPNCNotReady.Error()
}
}
Expand All @@ -135,16 +134,16 @@ func (m *K8sSWIFTv2Middleware) validateIPConfigsRequest(ctx context.Context, req
}

// getIPConfig returns the pod's SWIFT V2 IP configuration.
func (m *K8sSWIFTv2Middleware) getIPConfig(ctx context.Context, podInfo cns.PodInfo) (cns.PodIpInfo, error) {
func (k *K8sSWIFTv2Middleware) getIPConfig(ctx context.Context, podInfo cns.PodInfo) (cns.PodIpInfo, error) {
// Check if the MTPNC CRD exists for the pod, if not, return error
mtpnc := v1alpha1.MultitenantPodNetworkConfig{}
mtpncNamespacedName := k8stypes.NamespacedName{Namespace: podInfo.Namespace(), Name: podInfo.Name()}
if err := m.Cli.Get(ctx, mtpncNamespacedName, &mtpnc); err != nil {
if err := k.Cli.Get(ctx, mtpncNamespacedName, &mtpnc); err != nil {
return cns.PodIpInfo{}, errors.Wrapf(err, "failed to get pod's mtpnc from cache")
}

// Check if the MTPNC CRD is ready. If one of the fields is empty, return error
if mtpnc.Status.PrimaryIP == "" || mtpnc.Status.MacAddress == "" || mtpnc.Status.NCID == "" || mtpnc.Status.GatewayIP == "" {
if !mtpnc.IsReady() {
return cns.PodIpInfo{}, errMTPNCNotReady
}
logger.Printf("[SWIFTv2Middleware] mtpnc for pod %s is : %+v", podInfo.Name(), mtpnc)
Expand All @@ -166,119 +165,10 @@ func (m *K8sSWIFTv2Middleware) getIPConfig(ctx context.Context, podInfo cns.PodI
PrefixLength: uint8(prefixSize),
},
MacAddress: mtpnc.Status.MacAddress,
NICType: cns.DelegatedVMNIC,
NICType: mtpnc.Status.NICType,
SkipDefaultRoutes: false,
// InterfaceName is empty for DelegatedVMNIC
}

return podIPInfo, nil
}

// setRoutes sets the routes for podIPInfo used in SWIFT V2 scenario.
func (m *K8sSWIFTv2Middleware) setRoutes(podIPInfo *cns.PodIpInfo) error {
logger.Printf("[SWIFTv2Middleware] set routes for pod with nic type : %s", podIPInfo.NICType)
podIPInfo.Routes = []cns.Route{}
switch podIPInfo.NICType {
case cns.DelegatedVMNIC:
virtualGWRoute := cns.Route{
IPAddress: fmt.Sprintf("%s/%d", virtualGW, prefixLength),
}
// default route via SWIFT v2 interface
route := cns.Route{
IPAddress: "0.0.0.0/0",
GatewayIPAddress: virtualGW,
}
podIPInfo.Routes = []cns.Route{virtualGWRoute, route}
case cns.InfraNIC:
// Get and parse infraVNETCIDRs from env
infraVNETCIDRs, err := configuration.InfraVNETCIDRs()
if err != nil {
return errors.Wrapf(err, "failed to get infraVNETCIDRs from env")
}
infraVNETCIDRsv4, infraVNETCIDRsv6, err := utils.ParseCIDRs(infraVNETCIDRs)
if err != nil {
return errors.Wrapf(err, "failed to parse infraVNETCIDRs")
}

// Get and parse podCIDRs from env
podCIDRs, err := configuration.PodCIDRs()
if err != nil {
return errors.Wrapf(err, "failed to get podCIDRs from env")
}
podCIDRsV4, podCIDRv6, err := utils.ParseCIDRs(podCIDRs)
if err != nil {
return errors.Wrapf(err, "failed to parse podCIDRs")
}

// Get and parse serviceCIDRs from env
serviceCIDRs, err := configuration.ServiceCIDRs()
if err != nil {
return errors.Wrapf(err, "failed to get serviceCIDRs from env")
}
serviceCIDRsV4, serviceCIDRsV6, err := utils.ParseCIDRs(serviceCIDRs)
if err != nil {
return errors.Wrapf(err, "failed to parse serviceCIDRs")
}
// Check if the podIPInfo is IPv4 or IPv6
ip, err := netip.ParseAddr(podIPInfo.PodIPConfig.IPAddress)
if err != nil {
return errors.Wrapf(err, "failed to parse podIPConfig IP address %s", podIPInfo.PodIPConfig.IPAddress)
}
if ip.Is4() {
// routes for IPv4 podCIDR traffic
for _, podCIDRv4 := range podCIDRsV4 {
podCIDRv4Route := cns.Route{
IPAddress: podCIDRv4,
GatewayIPAddress: overlayGatewayv4,
}
podIPInfo.Routes = append(podIPInfo.Routes, podCIDRv4Route)
}
// route for IPv4 serviceCIDR traffic
for _, serviceCIDRv4 := range serviceCIDRsV4 {
serviceCIDRv4Route := cns.Route{
IPAddress: serviceCIDRv4,
GatewayIPAddress: overlayGatewayv4,
}
podIPInfo.Routes = append(podIPInfo.Routes, serviceCIDRv4Route)
}
// route for IPv4 infraVNETCIDR traffic
for _, infraVNETCIDRv4 := range infraVNETCIDRsv4 {
infraVNETCIDRv4Route := cns.Route{
IPAddress: infraVNETCIDRv4,
GatewayIPAddress: overlayGatewayv4,
}
podIPInfo.Routes = append(podIPInfo.Routes, infraVNETCIDRv4Route)
}
} else {
// routes for IPv6 podCIDR traffic
for _, podCIDRv6 := range podCIDRv6 {
podCIDRv6Route := cns.Route{
IPAddress: podCIDRv6,
GatewayIPAddress: overlayGatewayV6,
}
podIPInfo.Routes = append(podIPInfo.Routes, podCIDRv6Route)
}
// route for IPv6 serviceCIDR traffic
for _, serviceCIDRv6 := range serviceCIDRsV6 {
serviceCIDRv6Route := cns.Route{
IPAddress: serviceCIDRv6,
GatewayIPAddress: overlayGatewayV6,
}
podIPInfo.Routes = append(podIPInfo.Routes, serviceCIDRv6Route)
}
// route for IPv6 infraVNETCIDR traffic
for _, infraVNETCIDRv6 := range infraVNETCIDRsv6 {
infraVNETCIDRv6Route := cns.Route{
IPAddress: infraVNETCIDRv6,
GatewayIPAddress: overlayGatewayV6,
}
podIPInfo.Routes = append(podIPInfo.Routes, infraVNETCIDRv6Route)
}
}
podIPInfo.SkipDefaultRoutes = true
case cns.BackendNIC:
default:
return errInvalidSWIFTv2NICType
}
return nil
}
98 changes: 98 additions & 0 deletions cns/middlewares/k8sSwiftV2_linux.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,98 @@
package middlewares

import (
"fmt"
"net/netip"

"github.com/Azure/azure-container-networking/cns"
"github.com/Azure/azure-container-networking/cns/configuration"
"github.com/Azure/azure-container-networking/cns/logger"
"github.com/Azure/azure-container-networking/cns/middlewares/utils"
"github.com/pkg/errors"
)

// setRoutes sets the routes for podIPInfo used in SWIFT V2 scenario.
func (k *K8sSWIFTv2Middleware) setRoutes(podIPInfo *cns.PodIpInfo) error {
logger.Printf("[SWIFTv2Middleware] set routes for pod with nic type : %s", podIPInfo.NICType)
podIPInfo.Routes = []cns.Route{}
switch podIPInfo.NICType {
case cns.DelegatedVMNIC:
virtualGWRoute := cns.Route{
IPAddress: fmt.Sprintf("%s/%d", virtualGW, prefixLength),
}
// default route via SWIFT v2 interface
route := cns.Route{
IPAddress: "0.0.0.0/0",
GatewayIPAddress: virtualGW,
}
podIPInfo.Routes = []cns.Route{virtualGWRoute, route}
case cns.InfraNIC:
// Get and parse infraVNETCIDRs from env
infraVNETCIDRs, err := configuration.InfraVNETCIDRs()
if err != nil {
return errors.Wrapf(err, "failed to get infraVNETCIDRs from env")
}
infraVNETCIDRsv4, infraVNETCIDRsv6, err := utils.ParseCIDRs(infraVNETCIDRs)
if err != nil {
return errors.Wrapf(err, "failed to parse infraVNETCIDRs")
}

// Get and parse podCIDRs from env
podCIDRs, err := configuration.PodCIDRs()
if err != nil {
return errors.Wrapf(err, "failed to get podCIDRs from env")
}
podCIDRsV4, podCIDRv6, err := utils.ParseCIDRs(podCIDRs)
if err != nil {
return errors.Wrapf(err, "failed to parse podCIDRs")
}

// Get and parse serviceCIDRs from env
serviceCIDRs, err := configuration.ServiceCIDRs()
if err != nil {
return errors.Wrapf(err, "failed to get serviceCIDRs from env")
}
serviceCIDRsV4, serviceCIDRsV6, err := utils.ParseCIDRs(serviceCIDRs)
if err != nil {
return errors.Wrapf(err, "failed to parse serviceCIDRs")
}
// Check if the podIPInfo is IPv4 or IPv6
ip, err := netip.ParseAddr(podIPInfo.PodIPConfig.IPAddress)
if err != nil {
return errors.Wrapf(err, "failed to parse podIPConfig IP address %s", podIPInfo.PodIPConfig.IPAddress)
}
if ip.Is4() {
// routes for IPv4 podCIDR traffic
addRoutes(&podIPInfo.Routes, podCIDRsV4, overlayGatewayv4)
// route for IPv4 serviceCIDR traffic
addRoutes(&podIPInfo.Routes, serviceCIDRsV4, overlayGatewayv4)
// route for IPv4 infraVNETCIDR traffic
addRoutes(&podIPInfo.Routes, infraVNETCIDRsv4, overlayGatewayv4)
} else {
// routes for IPv6 podCIDR traffic
addRoutes(&podIPInfo.Routes, podCIDRv6, overlayGatewayV6)
// route for IPv6 serviceCIDR traffic
addRoutes(&podIPInfo.Routes, serviceCIDRsV6, overlayGatewayV6)
// route for IPv6 infraVNETCIDR traffic
addRoutes(&podIPInfo.Routes, infraVNETCIDRsv6, overlayGatewayV6)
}
podIPInfo.SkipDefaultRoutes = true
case cns.NodeNetworkInterfaceBackendNIC:
// TODO: Set routes for NodeNetworkInterfaceBackendNIC
case cns.NodeNetworkInterfaceAccelnetFrontendNIC:
// TODO: Set routes for NodeNetworkInterfaceAccelnetFrontendNIC
default:
return errInvalidSWIFTv2NICType
}
return nil
}

func addRoutes(routes *[]cns.Route, cidrs []string, gatewayIP string) {
for _, cidr := range cidrs {
route := cns.Route{
IPAddress: cidr,
GatewayIPAddress: gatewayIP,
}
*routes = append(*routes, route)
}
}
12 changes: 12 additions & 0 deletions cns/middlewares/k8sSwiftV2_windows.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
package middlewares

import (
"github.com/Azure/azure-container-networking/cns"
"github.com/Azure/azure-container-networking/cns/logger"
)

// setRoutes sets the routes for podIPInfo used in SWIFT V2 scenario. This is a no-op as route setting is not applicable for Windows.
func (k *K8sSWIFTv2Middleware) setRoutes(_ *cns.PodIpInfo) error {
logger.Printf("[SWIFTv2Middleware] setRoutes is a no-op on Windows")
return nil
}
3 changes: 3 additions & 0 deletions crd/multitenancy/api/v1alpha1/multitenantpodnetworkconfig.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
package v1alpha1

import (
"github.com/Azure/azure-container-networking/cns"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

Expand Down Expand Up @@ -61,6 +62,8 @@ type MultitenantPodNetworkConfigStatus struct {
MacAddress string `json:"macAddress,omitempty"`
// Gateway IP
GatewayIP string `json:"gatewayIP,omitempty"`
// NICType specifies the type of NIC to enable appropriate programming by CNS/CNI
NICType cns.NICType `json:"NICType,omitempty"`
}

func init() {
Expand Down
6 changes: 6 additions & 0 deletions crd/multitenancy/api/v1alpha1/utils.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
package v1alpha1

// IsReady checks if all the required fields in the MTPNC status are populated
func (m *MultitenantPodNetworkConfig) IsReady() bool {
return m.Status.PrimaryIP != "" && m.Status.MacAddress != "" && m.Status.NCID != "" && m.Status.GatewayIP != "" && m.Status.NICType != ""
}

0 comments on commit aabae63

Please sign in to comment.