From ace96ff5c2fce9a09613a0c60bf8f9ad9d20119e Mon Sep 17 00:00:00 2001 From: Quang Nguyen Date: Thu, 27 Jul 2023 22:09:30 +0000 Subject: [PATCH 01/85] fix overlay IPAM not reporting version --- Makefile | 2 +- .../{buildinfo/buildinfo.go => buildversion/buildversion.go} | 4 ++-- azure-ipam/ipam.go | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) rename azure-ipam/internal/{buildinfo/buildinfo.go => buildversion/buildversion.go} (74%) diff --git a/Makefile b/Makefile index 316523d6c0..80ffe737a9 100644 --- a/Makefile +++ b/Makefile @@ -173,7 +173,7 @@ zapai-version: ## prints the zapai version # Build the delegated IPAM plugin binary. azure-ipam-binary: - cd $(AZURE_IPAM_DIR) && CGO_ENABLED=0 go build -v -o $(AZURE_IPAM_BUILD_DIR)/azure-ipam$(EXE_EXT) -ldflags "-X main.version=$(AZURE_IPAM_VERSION)" -gcflags="-dwarflocationlists=true" + cd $(AZURE_IPAM_DIR) && CGO_ENABLED=0 go build -v -o $(AZURE_IPAM_BUILD_DIR)/azure-ipam$(EXE_EXT) -ldflags "-X buildversion.BuildVersion=$(AZURE_IPAM_VERSION)" -gcflags="-dwarflocationlists=true" # Build the Azure CNM binary. cnm-binary: diff --git a/azure-ipam/internal/buildinfo/buildinfo.go b/azure-ipam/internal/buildversion/buildversion.go similarity index 74% rename from azure-ipam/internal/buildinfo/buildinfo.go rename to azure-ipam/internal/buildversion/buildversion.go index e26b8d2f8e..e25cccd172 100644 --- a/azure-ipam/internal/buildinfo/buildinfo.go +++ b/azure-ipam/internal/buildversion/buildversion.go @@ -1,6 +1,6 @@ -package buildinfo +package buildversion // this will be populate by the Go compiler via // the -ldflags, which insert dynamic information // into the binary at build time -var Version string +var BuildVersion string diff --git a/azure-ipam/ipam.go b/azure-ipam/ipam.go index b3803a3c7d..da7c23ac7e 100644 --- a/azure-ipam/ipam.go +++ b/azure-ipam/ipam.go @@ -6,7 +6,7 @@ import ( "io" "net" - "github.com/Azure/azure-container-networking/azure-ipam/internal/buildinfo" + "github.com/Azure/azure-container-networking/azure-ipam/internal/buildversion" "github.com/Azure/azure-container-networking/azure-ipam/ipconfig" "github.com/Azure/azure-container-networking/cns" cniSkel "github.com/containernetworking/cni/pkg/skel" @@ -36,7 +36,7 @@ type cnsClient interface { func NewPlugin(logger *zap.Logger, c cnsClient, out io.Writer) (*IPAMPlugin, error) { plugin := &IPAMPlugin{ Name: pluginName, - Version: buildinfo.Version, + Version: buildversion.BuildVersion, logger: logger, out: out, cnsClient: c, From f1b5dcbe77fb07a7e0cda4377f4ba9e3f3e688d1 Mon Sep 17 00:00:00 2001 From: Quang Nguyen Date: Mon, 31 Jul 2023 22:09:59 +0000 Subject: [PATCH 02/85] revert file and var naming, add correct path to makefile --- Makefile | 2 +- .../{buildversion/buildversion.go => buildinfo/buildinfo.go} | 4 ++-- azure-ipam/ipam.go | 4 ++-- azure-ipam/main.go | 3 +++ 4 files changed, 8 insertions(+), 5 deletions(-) rename azure-ipam/internal/{buildversion/buildversion.go => buildinfo/buildinfo.go} (74%) diff --git a/Makefile b/Makefile index 80ffe737a9..132a0d39fb 100644 --- a/Makefile +++ b/Makefile @@ -173,7 +173,7 @@ zapai-version: ## prints the zapai version # Build the delegated IPAM plugin binary. azure-ipam-binary: - cd $(AZURE_IPAM_DIR) && CGO_ENABLED=0 go build -v -o $(AZURE_IPAM_BUILD_DIR)/azure-ipam$(EXE_EXT) -ldflags "-X buildversion.BuildVersion=$(AZURE_IPAM_VERSION)" -gcflags="-dwarflocationlists=true" + cd $(AZURE_IPAM_DIR) && CGO_ENABLED=0 go build -v -o $(AZURE_IPAM_BUILD_DIR)/azure-ipam$(EXE_EXT) -ldflags "-X github.com/Azure/azure-container-networking/azure-ipam/internal/buildinfo.Version=$(AZURE_IPAM_VERSION)" -gcflags="-dwarflocationlists=true" # Build the Azure CNM binary. cnm-binary: diff --git a/azure-ipam/internal/buildversion/buildversion.go b/azure-ipam/internal/buildinfo/buildinfo.go similarity index 74% rename from azure-ipam/internal/buildversion/buildversion.go rename to azure-ipam/internal/buildinfo/buildinfo.go index e25cccd172..e26b8d2f8e 100644 --- a/azure-ipam/internal/buildversion/buildversion.go +++ b/azure-ipam/internal/buildinfo/buildinfo.go @@ -1,6 +1,6 @@ -package buildversion +package buildinfo // this will be populate by the Go compiler via // the -ldflags, which insert dynamic information // into the binary at build time -var BuildVersion string +var Version string diff --git a/azure-ipam/ipam.go b/azure-ipam/ipam.go index da7c23ac7e..b3803a3c7d 100644 --- a/azure-ipam/ipam.go +++ b/azure-ipam/ipam.go @@ -6,7 +6,7 @@ import ( "io" "net" - "github.com/Azure/azure-container-networking/azure-ipam/internal/buildversion" + "github.com/Azure/azure-container-networking/azure-ipam/internal/buildinfo" "github.com/Azure/azure-container-networking/azure-ipam/ipconfig" "github.com/Azure/azure-container-networking/cns" cniSkel "github.com/containernetworking/cni/pkg/skel" @@ -36,7 +36,7 @@ type cnsClient interface { func NewPlugin(logger *zap.Logger, c cnsClient, out io.Writer) (*IPAMPlugin, error) { plugin := &IPAMPlugin{ Name: pluginName, - Version: buildversion.BuildVersion, + Version: buildinfo.Version, logger: logger, out: out, cnsClient: c, diff --git a/azure-ipam/main.go b/azure-ipam/main.go index d040da9d91..d6194ce962 100644 --- a/azure-ipam/main.go +++ b/azure-ipam/main.go @@ -4,6 +4,7 @@ import ( "log" "os" + "github.com/Azure/azure-container-networking/azure-ipam/internal/buildinfo" "github.com/Azure/azure-container-networking/azure-ipam/logger" cnsclient "github.com/Azure/azure-container-networking/cns/client" "github.com/containernetworking/cni/pkg/skel" @@ -48,6 +49,8 @@ func executePlugin() error { return errors.Wrapf(err, "failed to create IPAM plugin") } + bv.BuildVersion = buildinfo.Version + // Execute CNI plugin cniErr := skel.PluginMainWithError(plugin.CmdAdd, plugin.CmdCheck, plugin.CmdDel, version.All, bv.BuildString(pluginName)) if cniErr != nil { From 20988bd5863f24160c53738830fae5100860c7bb Mon Sep 17 00:00:00 2001 From: Quang Nguyen Date: Wed, 9 Aug 2023 20:17:02 +0000 Subject: [PATCH 03/85] proposal design for multitenant IPAM flow --- cns/NetworkContainerContract.go | 9 ++++++++ cns/middlewares/multitenant_validator.go | 22 ++++++++++++++++++++ cns/restserver/ipam.go | 3 +++ cns/restserver/restserver.go | 8 ++++++++ cns/restserver/util.go | 26 ++++++++++++++++-------- 5 files changed, 60 insertions(+), 8 deletions(-) create mode 100644 cns/middlewares/multitenant_validator.go diff --git a/cns/NetworkContainerContract.go b/cns/NetworkContainerContract.go index 9d041a6cfe..f73f96c795 100644 --- a/cns/NetworkContainerContract.go +++ b/cns/NetworkContainerContract.go @@ -171,6 +171,8 @@ type PodInfo interface { Equals(PodInfo) bool // String implements string for logging PodInfos String() string + // IsMultitenant returns true if the pod is multitenant + IsMultitenant() bool } type KubernetesPodInfo struct { @@ -186,6 +188,7 @@ type podInfo struct { PodInfraContainerID string PodInterfaceID string Version podInfoScheme + Multitenant bool } func (p podInfo) String() string { @@ -239,6 +242,10 @@ func (p *podInfo) OrchestratorContext() (json.RawMessage, error) { return jsonContext, nil } +func (p *podInfo) IsMultitenant() bool { + return p.Multitenant +} + // NewPodInfo returns an implementation of PodInfo that returns the passed // configuration for their namesake functions. func NewPodInfo(infraContainerID, interfaceID, name, namespace string) PodInfo { @@ -276,6 +283,7 @@ func NewPodInfoFromIPConfigsRequest(req IPConfigsRequest) (PodInfo, error) { } p.(*podInfo).PodInfraContainerID = req.InfraContainerID p.(*podInfo).PodInterfaceID = req.PodInterfaceID + p.(*podInfo).Multitenant = req.Multitenant return p, nil } @@ -427,6 +435,7 @@ type IPConfigsRequest struct { InfraContainerID string `json:"infraContainerID"` OrchestratorContext json.RawMessage `json:"orchestratorContext"` Ifname string `json:"ifname"` // Used by delegated IPAM + Multitenant bool `json:"multitenant"` } // IPConfigResponse is used in CNS IPAM mode as a response to CNI ADD diff --git a/cns/middlewares/multitenant_validator.go b/cns/middlewares/multitenant_validator.go new file mode 100644 index 0000000000..2a21affeb9 --- /dev/null +++ b/cns/middlewares/multitenant_validator.go @@ -0,0 +1,22 @@ +package middlewares + +import ( + "github.com/Azure/azure-container-networking/cns" + "github.com/Azure/azure-container-networking/cns/types" +) + +type MultitenantValidator struct { + // TODO: implement + // need cached scoped client for pods +} + +func NewMultitenantValidator() *MultitenantValidator { + return &MultitenantValidator{} +} + +// validateMultitenantIPConfigsRequest validate whether the request is for a multitenant pod +func (v *MultitenantValidator) validateMultitenantIPConfigsRequest(ipConfigsRequest *cns.IPConfigsRequest) (types.ResponseCode, string) { + // TODO: if pod is multitenant, enrich the request with the multitenant flag + ipConfigsRequest.Multitenant = true + return types.Success, "" +} diff --git a/cns/restserver/ipam.go b/cns/restserver/ipam.go index d148692c66..b12472fbfb 100644 --- a/cns/restserver/ipam.go +++ b/cns/restserver/ipam.go @@ -132,6 +132,8 @@ func (service *HTTPRestService) requestIPConfigHandler(w http.ResponseWriter, r // Checks to make sure we return exactly 1 IP // If IPAM assigned more than 1 IP then we need to raise an error since this API can only return one IP and IPAM may have assigned more than one if len(ipConfigsResp.PodIPInfo) != 1 { + // TODO: if cns is running in a multitenant node, then this is fine, and we should return the whole list of PodIpInfo + // we send a response back saying that this API won't be able to return the amount of IPs needed to fulfill the request reserveResp := &cns.IPConfigResponse{ Response: cns.Response{ @@ -864,6 +866,7 @@ func requestIPConfigsHelper(service *HTTPRestService, req cns.IPConfigsRequest) return []cns.PodIpInfo{}, errors.Wrapf(err, "failed to parse IPConfigsRequest %v", req) } + // maybe adding the multitenant ip assignment somewhere around here? if podIPInfo, isExist, err := service.GetExistingIPConfig(podInfo); err != nil || isExist { return podIPInfo, err } diff --git a/cns/restserver/restserver.go b/cns/restserver/restserver.go index 70676c28e3..b893f334b1 100644 --- a/cns/restserver/restserver.go +++ b/cns/restserver/restserver.go @@ -51,6 +51,8 @@ type wireserverProxy interface { UnpublishNC(ctx context.Context, ncParams cns.NetworkContainerParameters, payload []byte) (*http.Response, error) } +type IPConfigValidator func(ipConfigsRequest *cns.IPConfigsRequest) (types.ResponseCode, string) + // HTTPRestService represents http listener for CNS - Container Networking Service. type HTTPRestService struct { *cns.Service @@ -74,6 +76,7 @@ type HTTPRestService struct { EndpointStateStore store.KeyValueStore cniConflistGenerator CNIConflistGenerator generateCNIConflistOnce sync.Once + ipConfigsValidators []IPConfigValidator } type CNIConflistGenerator interface { @@ -228,6 +231,11 @@ func (service *HTTPRestService) Init(config *common.ServiceConfig) error { return err } + // Adding ipConfigsValidators + service.ipConfigsValidators = []IPConfigValidator{service.validateDefaultIPConfigsRequest} + // if cns is running in multitenant mode, add the multitenant validator + // service.ipConfigsValidators = append(service.ipConfigsValidators, NewMultitenantValidator().validateMultitenantIPConfigsRequest) + // Add handlers. listener := service.Listener // default handlers diff --git a/cns/restserver/util.go b/cns/restserver/util.go index f8b4822a07..73c606ea77 100644 --- a/cns/restserver/util.go +++ b/cns/restserver/util.go @@ -769,14 +769,12 @@ func (service *HTTPRestService) SendNCSnapShotPeriodically(ctx context.Context, func (service *HTTPRestService) validateIPConfigsRequest( ipConfigsRequest cns.IPConfigsRequest, ) (cns.PodInfo, types.ResponseCode, string) { - if service.state.OrchestratorType != cns.KubernetesCRD && service.state.OrchestratorType != cns.Kubernetes { - return nil, types.UnsupportedOrchestratorType, "ReleaseIPConfig API supported only for kubernetes orchestrator" - } - - if ipConfigsRequest.OrchestratorContext == nil { - return nil, - types.EmptyOrchestratorContext, - fmt.Sprintf("OrchastratorContext is not set in the req: %+v", ipConfigsRequest) + // looping through all the ipconfigs request validators, if any validator fails, return the error + for _, validator := range service.ipConfigsValidators { + respCode, message := validator(&ipConfigsRequest) + if respCode != types.Success { + return nil, respCode, message + } } // retrieve podinfo from orchestrator context @@ -787,6 +785,18 @@ func (service *HTTPRestService) validateIPConfigsRequest( return podInfo, types.Success, "" } +// validateDefaultIPConfigsRequest validates the request for default IP configs request +func (service *HTTPRestService) validateDefaultIPConfigsRequest(ipConfigsRequest *cns.IPConfigsRequest) (types.ResponseCode, string) { + if service.state.OrchestratorType != cns.KubernetesCRD && service.state.OrchestratorType != cns.Kubernetes { + return types.UnsupportedOrchestratorType, "ReleaseIPConfig API supported only for kubernetes orchestrator" + } + + if ipConfigsRequest.OrchestratorContext == nil { + return types.EmptyOrchestratorContext, fmt.Sprintf("OrchastratorContext is not set in the req: %+v", ipConfigsRequest) + } + return types.Success, "" +} + // getPrimaryHostInterface returns the cached InterfaceInfo, if available, otherwise // queries the IMDS to get the primary interface info and caches it in the server state // before returning the result. From 2e6842f37d8d3aceac9358765a378c5c9cc6921f Mon Sep 17 00:00:00 2001 From: Quang Nguyen Date: Wed, 9 Aug 2023 20:28:51 +0000 Subject: [PATCH 04/85] change podipinfo + linter issue --- cns/NetworkContainerContract.go | 2 ++ cns/middlewares/multitenant_validator.go | 3 ++- cns/restserver/util.go | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/cns/NetworkContainerContract.go b/cns/NetworkContainerContract.go index f73f96c795..36b6d3768a 100644 --- a/cns/NetworkContainerContract.go +++ b/cns/NetworkContainerContract.go @@ -412,6 +412,8 @@ type PodIpInfo struct { PodIPConfig IPSubnet NetworkContainerPrimaryIPConfig IPConfiguration HostPrimaryIPInfo HostIPInfo + AddressType string // multitenant or default, maybe make this an enum later + MACAddress string } type HostIPInfo struct { diff --git a/cns/middlewares/multitenant_validator.go b/cns/middlewares/multitenant_validator.go index 2a21affeb9..ae31adaf05 100644 --- a/cns/middlewares/multitenant_validator.go +++ b/cns/middlewares/multitenant_validator.go @@ -15,7 +15,8 @@ func NewMultitenantValidator() *MultitenantValidator { } // validateMultitenantIPConfigsRequest validate whether the request is for a multitenant pod -func (v *MultitenantValidator) validateMultitenantIPConfigsRequest(ipConfigsRequest *cns.IPConfigsRequest) (types.ResponseCode, string) { +// nolint +func (v *MultitenantValidator) validateMultitenantIPConfigsRequest(ipConfigsRequest *cns.IPConfigsRequest) (respCode types.ResponseCode, message string) { // TODO: if pod is multitenant, enrich the request with the multitenant flag ipConfigsRequest.Multitenant = true return types.Success, "" diff --git a/cns/restserver/util.go b/cns/restserver/util.go index 73c606ea77..b5fd47d2bf 100644 --- a/cns/restserver/util.go +++ b/cns/restserver/util.go @@ -786,7 +786,7 @@ func (service *HTTPRestService) validateIPConfigsRequest( } // validateDefaultIPConfigsRequest validates the request for default IP configs request -func (service *HTTPRestService) validateDefaultIPConfigsRequest(ipConfigsRequest *cns.IPConfigsRequest) (types.ResponseCode, string) { +func (service *HTTPRestService) validateDefaultIPConfigsRequest(ipConfigsRequest *cns.IPConfigsRequest) (respCode types.ResponseCode, message string) { if service.state.OrchestratorType != cns.KubernetesCRD && service.state.OrchestratorType != cns.Kubernetes { return types.UnsupportedOrchestratorType, "ReleaseIPConfig API supported only for kubernetes orchestrator" } From 227fd89c2c39d0f0000e9a1c5caa093c3e9ce7bd Mon Sep 17 00:00:00 2001 From: Quang Nguyen Date: Thu, 10 Aug 2023 17:47:08 +0000 Subject: [PATCH 05/85] pointer issues for printf --- cns/restserver/ipam.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cns/restserver/ipam.go b/cns/restserver/ipam.go index b12472fbfb..14bf1b5755 100644 --- a/cns/restserver/ipam.go +++ b/cns/restserver/ipam.go @@ -43,7 +43,7 @@ func (service *HTTPRestService) requestIPConfigHandlerHelper(ipconfigsRequest cn return &cns.IPConfigsResponse{ Response: cns.Response{ ReturnCode: types.FailedToAllocateIPConfig, - Message: fmt.Sprintf("AllocateIPConfig failed: %v, IP config request is %s", err, ipconfigsRequest), + Message: fmt.Sprintf("AllocateIPConfig failed: %v, IP config request is %v", err, ipconfigsRequest), }, PodIPInfo: podIPInfo, }, err @@ -257,7 +257,7 @@ func (service *HTTPRestService) releaseIPConfigHandlerHelper(ipconfigsRequest cn ReturnCode: types.UnexpectedError, Message: err.Error(), } - return resp, fmt.Errorf("releaseIPConfigHandlerHelper remove endpoint state failed because %v, release IP config info %s", resp.Message, ipconfigsRequest) //nolint:goerr113 // return error + return resp, fmt.Errorf("releaseIPConfigHandlerHelper remove endpoint state failed because %v, release IP config info %+v", resp.Message, ipconfigsRequest) //nolint:goerr113 // return error } } @@ -265,7 +265,7 @@ func (service *HTTPRestService) releaseIPConfigHandlerHelper(ipconfigsRequest cn return &cns.Response{ ReturnCode: types.UnexpectedError, Message: err.Error(), - }, fmt.Errorf("releaseIPConfigHandlerHelper releaseIPConfigs failed because %v, release IP config info %s", returnMessage, ipconfigsRequest) //nolint:goerr113 // return error + }, fmt.Errorf("releaseIPConfigHandlerHelper releaseIPConfigs failed because %v, release IP config info %+v", returnMessage, ipconfigsRequest) //nolint:goerr113 // return error } return &cns.Response{ @@ -337,7 +337,7 @@ func (service *HTTPRestService) releaseIPConfigsHandler(w http.ResponseWriter, r ReturnCode: types.UnexpectedError, Message: err.Error(), } - logger.Errorf("releaseIPConfigsHandler decode failed because %v, release IP config info %s", resp.Message, ipconfigsRequest) + logger.Errorf("releaseIPConfigsHandler decode failed because %v, release IP config info %+v", resp.Message, ipconfigsRequest) w.Header().Set(cnsReturnCode, resp.ReturnCode.String()) err = service.Listener.Encode(w, &resp) logger.ResponseEx(service.Name, ipconfigsRequest, resp, resp.ReturnCode, err) From d5b00dbe53512d1a4df6611de3e559e5623fe62e Mon Sep 17 00:00:00 2001 From: Quang Nguyen Date: Tue, 15 Aug 2023 20:08:57 +0000 Subject: [PATCH 06/85] update IPAM branching --- cns/middlewares/multitenant.go | 50 ++++++++++++++++++++++++ cns/middlewares/multitenant_validator.go | 23 ----------- cns/restserver/ipam.go | 34 +++++++++++++++- cns/restserver/restserver.go | 22 ++++++----- cns/service/main.go | 6 +++ 5 files changed, 101 insertions(+), 34 deletions(-) create mode 100644 cns/middlewares/multitenant.go delete mode 100644 cns/middlewares/multitenant_validator.go diff --git a/cns/middlewares/multitenant.go b/cns/middlewares/multitenant.go new file mode 100644 index 0000000000..e0ddf4d1f6 --- /dev/null +++ b/cns/middlewares/multitenant.go @@ -0,0 +1,50 @@ +package middlewares + +import ( + "github.com/Azure/azure-container-networking/cns" + "github.com/Azure/azure-container-networking/cns/types" +) + +type IPConfigValidator func(ipConfigsRequest *cns.IPConfigsRequest) (types.ResponseCode, string) + +// Middleware interface for testing later on +type Middleware interface { + Validator() IPConfigValidator + GetMultitenantIPConfig(podInfo cns.PodInfo) (*cns.PodIpInfo, error) +} + +type MultitenantMiddleware struct { + // TODO: implement + // need cached scoped client for pods + // need client for MTPNC CRD for x-ref pods +} + +func NewMultitenantMiddleware() *MultitenantMiddleware { + return &MultitenantMiddleware{} +} + +// Return the validator function for the middleware +func (m *MultitenantMiddleware) Validator() IPConfigValidator { + return m.validateMultitenantIPConfigsRequest +} + +// validateMultitenantIPConfigsRequest validate whether the request is for a multitenant pod +// nolint +func (m *MultitenantMiddleware) validateMultitenantIPConfigsRequest(ipConfigsRequest *cns.IPConfigsRequest) (respCode types.ResponseCode, message string) { + /** + TODO: + - Check if pod is multitenant, enrich the request with the multitenant flag + **/ + ipConfigsRequest.Multitenant = true + return types.Success, "" +} + +// nolint +// GetMultitenantIPConfig returns the IP config for a multitenant pod from the MTPNC CRD +func (m *MultitenantMiddleware) GetMultitenantIPConfig(podInfo cns.PodInfo) (*cns.PodIpInfo, error) { + /** + TODO: + - Check if the MTPNC CRD exists for the pod, if not, return error + **/ + return nil, nil +} diff --git a/cns/middlewares/multitenant_validator.go b/cns/middlewares/multitenant_validator.go deleted file mode 100644 index ae31adaf05..0000000000 --- a/cns/middlewares/multitenant_validator.go +++ /dev/null @@ -1,23 +0,0 @@ -package middlewares - -import ( - "github.com/Azure/azure-container-networking/cns" - "github.com/Azure/azure-container-networking/cns/types" -) - -type MultitenantValidator struct { - // TODO: implement - // need cached scoped client for pods -} - -func NewMultitenantValidator() *MultitenantValidator { - return &MultitenantValidator{} -} - -// validateMultitenantIPConfigsRequest validate whether the request is for a multitenant pod -// nolint -func (v *MultitenantValidator) validateMultitenantIPConfigsRequest(ipConfigsRequest *cns.IPConfigsRequest) (respCode types.ResponseCode, message string) { - // TODO: if pod is multitenant, enrich the request with the multitenant flag - ipConfigsRequest.Multitenant = true - return types.Success, "" -} diff --git a/cns/restserver/ipam.go b/cns/restserver/ipam.go index 14bf1b5755..d9a1392646 100644 --- a/cns/restserver/ipam.go +++ b/cns/restserver/ipam.go @@ -35,6 +35,34 @@ func (service *HTTPRestService) requestIPConfigHandlerHelper(ipconfigsRequest cn }, errors.New("failed to validate ip config request") } + var MTNpodIPInfo *cns.PodIpInfo + + // Request is for multitenant pod + if ipconfigsRequest.Multitenant { + // Multitenant middleware not set + if service.MultitenantMiddleware == nil { + return &cns.IPConfigsResponse{ + Response: cns.Response{ + ReturnCode: types.FailedToAllocateIPConfig, + Message: fmt.Sprintf("AllocateIPConfig failed: %v, IP config request is %v", errors.New("request is for multitenant pod but multitenant middleware is nil"), ipconfigsRequest), + }, + PodIPInfo: []cns.PodIpInfo{}, + }, errors.New("request is for multitenant pod but multitenant middleware is nil") + } + // If pod is multitenant and we failed to grab its MTPNC IP config, return error immediately + podIPInfo, err := service.MultitenantMiddleware.GetMultitenantIPConfig(podInfo) + if err != nil { + return &cns.IPConfigsResponse{ + Response: cns.Response{ + ReturnCode: types.FailedToAllocateIPConfig, + Message: fmt.Sprintf("AllocateIPConfig failed: %v, IP config request is %v", err, ipconfigsRequest), + }, + PodIPInfo: []cns.PodIpInfo{}, + }, errors.Wrapf(err, "failed to get multitenant IP config %v", ipconfigsRequest) + } + MTNpodIPInfo = podIPInfo + } + // record a pod requesting an IP service.podsPendingIPAssignment.Push(podInfo.Key()) @@ -71,6 +99,11 @@ func (service *HTTPRestService) requestIPConfigHandlerHelper(ipconfigsRequest cn } } + // Adding MTNpodIPInfo to the response, skipping over updateEndpointState since not sure if we have to update endpoint state for MTNpodIPInfo + if MTNpodIPInfo != nil { + podIPInfo = append(podIPInfo, *MTNpodIPInfo) + } + return &cns.IPConfigsResponse{ Response: cns.Response{ ReturnCode: types.Success, @@ -866,7 +899,6 @@ func requestIPConfigsHelper(service *HTTPRestService, req cns.IPConfigsRequest) return []cns.PodIpInfo{}, errors.Wrapf(err, "failed to parse IPConfigsRequest %v", req) } - // maybe adding the multitenant ip assignment somewhere around here? if podIPInfo, isExist, err := service.GetExistingIPConfig(podInfo); err != nil || isExist { return podIPInfo, err } diff --git a/cns/restserver/restserver.go b/cns/restserver/restserver.go index b893f334b1..9ce9ac623f 100644 --- a/cns/restserver/restserver.go +++ b/cns/restserver/restserver.go @@ -13,6 +13,7 @@ import ( "github.com/Azure/azure-container-networking/cns/dockerclient" "github.com/Azure/azure-container-networking/cns/ipamclient" "github.com/Azure/azure-container-networking/cns/logger" + "github.com/Azure/azure-container-networking/cns/middlewares" "github.com/Azure/azure-container-networking/cns/networkcontainers" "github.com/Azure/azure-container-networking/cns/routes" "github.com/Azure/azure-container-networking/cns/types" @@ -30,10 +31,8 @@ import ( // All helper/utility functions - util.go // Constants - const.go -var ( - // Named Lock for accessing different states in httpRestServiceState - namedLock = acn.InitNamedLock() -) +// Named Lock for accessing different states in httpRestServiceState +var namedLock = acn.InitNamedLock() type interfaceGetter interface { GetInterfaces(ctx context.Context) (*wireserver.GetInterfacesResult, error) @@ -51,8 +50,6 @@ type wireserverProxy interface { UnpublishNC(ctx context.Context, ncParams cns.NetworkContainerParameters, payload []byte) (*http.Response, error) } -type IPConfigValidator func(ipConfigsRequest *cns.IPConfigsRequest) (types.ResponseCode, string) - // HTTPRestService represents http listener for CNS - Container Networking Service. type HTTPRestService struct { *cns.Service @@ -76,7 +73,8 @@ type HTTPRestService struct { EndpointStateStore store.KeyValueStore cniConflistGenerator CNIConflistGenerator generateCNIConflistOnce sync.Once - ipConfigsValidators []IPConfigValidator + ipConfigsValidators []middlewares.IPConfigValidator + MultitenantMiddleware middlewares.Middleware } type CNIConflistGenerator interface { @@ -232,9 +230,7 @@ func (service *HTTPRestService) Init(config *common.ServiceConfig) error { } // Adding ipConfigsValidators - service.ipConfigsValidators = []IPConfigValidator{service.validateDefaultIPConfigsRequest} - // if cns is running in multitenant mode, add the multitenant validator - // service.ipConfigsValidators = append(service.ipConfigsValidators, NewMultitenantValidator().validateMultitenantIPConfigsRequest) + service.ipConfigsValidators = []middlewares.IPConfigValidator{service.validateDefaultIPConfigsRequest} // Add handlers. listener := service.Listener @@ -357,3 +353,9 @@ func (service *HTTPRestService) MustGenerateCNIConflistOnce() { } }) } + +func (service *HTTPRestService) AttachMultitenantMiddleware(middleware middlewares.Middleware) { + service.MultitenantMiddleware = middleware + // add multitenant ipconfig validator function + service.ipConfigsValidators = append(service.ipConfigsValidators, middleware.Validator()) +} diff --git a/cns/service/main.go b/cns/service/main.go index 44946fba35..8f9759478e 100644 --- a/cns/service/main.go +++ b/cns/service/main.go @@ -33,6 +33,7 @@ import ( cssctrl "github.com/Azure/azure-container-networking/cns/kubecontroller/clustersubnetstate" nncctrl "github.com/Azure/azure-container-networking/cns/kubecontroller/nodenetworkconfig" "github.com/Azure/azure-container-networking/cns/logger" + "github.com/Azure/azure-container-networking/cns/middlewares" "github.com/Azure/azure-container-networking/cns/multitenantcontroller" "github.com/Azure/azure-container-networking/cns/multitenantcontroller/multitenantoperator" "github.com/Azure/azure-container-networking/cns/restserver" @@ -789,6 +790,11 @@ func main() { if cnsconfig.EnablePprof { httpRestService.RegisterPProfEndpoints() } + // if SWIFT v2 is enabled on CNS, attach multitenant middleware to rest service + if cnsconfig.EnableSwiftV2 { + multitenantMiddleware := middlewares.NewMultitenantMiddleware() + httpRestService.AttachMultitenantMiddleware(multitenantMiddleware) + } err = httpRestService.Start(&config) if err != nil { From e53f1e7bbfc01231d8a49191a0f011e2dcbe1b45 Mon Sep 17 00:00:00 2001 From: Quang Nguyen Date: Tue, 15 Aug 2023 20:13:44 +0000 Subject: [PATCH 07/85] remove comments --- cns/restserver/ipam.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/cns/restserver/ipam.go b/cns/restserver/ipam.go index d9a1392646..e34997e5f0 100644 --- a/cns/restserver/ipam.go +++ b/cns/restserver/ipam.go @@ -165,8 +165,6 @@ func (service *HTTPRestService) requestIPConfigHandler(w http.ResponseWriter, r // Checks to make sure we return exactly 1 IP // If IPAM assigned more than 1 IP then we need to raise an error since this API can only return one IP and IPAM may have assigned more than one if len(ipConfigsResp.PodIPInfo) != 1 { - // TODO: if cns is running in a multitenant node, then this is fine, and we should return the whole list of PodIpInfo - // we send a response back saying that this API won't be able to return the amount of IPs needed to fulfill the request reserveResp := &cns.IPConfigResponse{ Response: cns.Response{ From f99525c58ef19c679de8ec1cb59220617eecbb31 Mon Sep 17 00:00:00 2001 From: Quang Nguyen Date: Fri, 18 Aug 2023 18:27:16 +0000 Subject: [PATCH 08/85] pod client placeholder --- cns/api.go | 9 +++++++++ cns/middlewares/multitenant.go | 18 +++++++----------- cns/restserver/restserver.go | 9 ++++----- cns/service/main.go | 14 +++++++++----- 4 files changed, 29 insertions(+), 21 deletions(-) diff --git a/cns/api.go b/cns/api.go index 705907b238..8595d617ee 100644 --- a/cns/api.go +++ b/cns/api.go @@ -48,8 +48,17 @@ type HTTPService interface { GetPendingReleaseIPConfigs() []IPConfigurationStatus GetPodIPConfigState() map[string]IPConfigurationStatus MarkIPAsPendingRelease(numberToMark int) (map[string]IPConfigurationStatus, error) + AttachMultitenantMiddleware(middleware MultitenantMiddleware) } +// Middleware interface for testing later on +type MultitenantMiddleware interface { + Validator() IPConfigValidator + GetMultitenantIPConfig(podInfo PodInfo) (*PodIpInfo, error) +} + +type IPConfigValidator func(ipConfigsRequest *IPConfigsRequest) (types.ResponseCode, string) + // This is used for KubernetesCRD orchestrator Type where NC has multiple ips. // This struct captures the state for SecondaryIPs associated to a given NC type IPConfigurationStatus struct { diff --git a/cns/middlewares/multitenant.go b/cns/middlewares/multitenant.go index e0ddf4d1f6..94ad1956c0 100644 --- a/cns/middlewares/multitenant.go +++ b/cns/middlewares/multitenant.go @@ -3,28 +3,24 @@ package middlewares import ( "github.com/Azure/azure-container-networking/cns" "github.com/Azure/azure-container-networking/cns/types" + "sigs.k8s.io/controller-runtime/pkg/client" ) -type IPConfigValidator func(ipConfigsRequest *cns.IPConfigsRequest) (types.ResponseCode, string) - -// Middleware interface for testing later on -type Middleware interface { - Validator() IPConfigValidator - GetMultitenantIPConfig(podInfo cns.PodInfo) (*cns.PodIpInfo, error) -} - type MultitenantMiddleware struct { // TODO: implement // need cached scoped client for pods // need client for MTPNC CRD for x-ref pods + cli client.Client } -func NewMultitenantMiddleware() *MultitenantMiddleware { - return &MultitenantMiddleware{} +func NewMultitenantMiddleware(cli client.Client) *MultitenantMiddleware { + return &MultitenantMiddleware{ + cli: cli, + } } // Return the validator function for the middleware -func (m *MultitenantMiddleware) Validator() IPConfigValidator { +func (m *MultitenantMiddleware) Validator() cns.IPConfigValidator { return m.validateMultitenantIPConfigsRequest } diff --git a/cns/restserver/restserver.go b/cns/restserver/restserver.go index 9ce9ac623f..ecef0123da 100644 --- a/cns/restserver/restserver.go +++ b/cns/restserver/restserver.go @@ -13,7 +13,6 @@ import ( "github.com/Azure/azure-container-networking/cns/dockerclient" "github.com/Azure/azure-container-networking/cns/ipamclient" "github.com/Azure/azure-container-networking/cns/logger" - "github.com/Azure/azure-container-networking/cns/middlewares" "github.com/Azure/azure-container-networking/cns/networkcontainers" "github.com/Azure/azure-container-networking/cns/routes" "github.com/Azure/azure-container-networking/cns/types" @@ -73,8 +72,8 @@ type HTTPRestService struct { EndpointStateStore store.KeyValueStore cniConflistGenerator CNIConflistGenerator generateCNIConflistOnce sync.Once - ipConfigsValidators []middlewares.IPConfigValidator - MultitenantMiddleware middlewares.Middleware + ipConfigsValidators []cns.IPConfigValidator + MultitenantMiddleware cns.MultitenantMiddleware } type CNIConflistGenerator interface { @@ -230,7 +229,7 @@ func (service *HTTPRestService) Init(config *common.ServiceConfig) error { } // Adding ipConfigsValidators - service.ipConfigsValidators = []middlewares.IPConfigValidator{service.validateDefaultIPConfigsRequest} + service.ipConfigsValidators = []cns.IPConfigValidator{service.validateDefaultIPConfigsRequest} // Add handlers. listener := service.Listener @@ -354,7 +353,7 @@ func (service *HTTPRestService) MustGenerateCNIConflistOnce() { }) } -func (service *HTTPRestService) AttachMultitenantMiddleware(middleware middlewares.Middleware) { +func (service *HTTPRestService) AttachMultitenantMiddleware(middleware cns.MultitenantMiddleware) { service.MultitenantMiddleware = middleware // add multitenant ipconfig validator function service.ipConfigsValidators = append(service.ipConfigsValidators, middleware.Validator()) diff --git a/cns/service/main.go b/cns/service/main.go index 8f9759478e..81f52d54d5 100644 --- a/cns/service/main.go +++ b/cns/service/main.go @@ -55,6 +55,7 @@ import ( "github.com/avast/retry-go/v3" "github.com/pkg/errors" "go.uber.org/zap" + v1 "k8s.io/api/apps/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" @@ -790,11 +791,6 @@ func main() { if cnsconfig.EnablePprof { httpRestService.RegisterPProfEndpoints() } - // if SWIFT v2 is enabled on CNS, attach multitenant middleware to rest service - if cnsconfig.EnableSwiftV2 { - multitenantMiddleware := middlewares.NewMultitenantMiddleware() - httpRestService.AttachMultitenantMiddleware(multitenantMiddleware) - } err = httpRestService.Start(&config) if err != nil { @@ -1205,6 +1201,10 @@ func InitializeCRDState(ctx context.Context, httpRestService cns.HTTPService, cn if err = v1alpha1.AddToScheme(crdSchemes); err != nil { return errors.Wrap(err, "failed to add clustersubnetstate/v1alpha1 to scheme") } + if err = v1.AddToScheme(crdSchemes); err != nil { + return errors.Wrap(err, "failed to add core/v1 to scheme") + } + manager, err := ctrl.NewManager(kubeConfig, ctrl.Options{ Scheme: crdSchemes, MetricsBindAddress: "0", @@ -1245,6 +1245,10 @@ func InitializeCRDState(ctx context.Context, httpRestService cns.HTTPService, cn if _, ok := node.Labels[configuration.LabelSwiftV2]; ok { cnsconfig.EnableSwiftV2 = true // TODO(rbtr): create the NodeInfo for Swift V2 + // if SWIFT v2 is enabled on CNS, attach multitenant middleware to rest service + multitenantMiddleware := middlewares.NewMultitenantMiddleware(manager.GetClient()) + httpRestService.AttachMultitenantMiddleware(multitenantMiddleware) + } // get CNS Node IP to compare NC Node IP with this Node IP to ensure NCs were created for this node From c7f1bca88598949404952aaeca4445325d5c9574 Mon Sep 17 00:00:00 2001 From: Quang Nguyen Date: Fri, 18 Aug 2023 18:43:30 +0000 Subject: [PATCH 09/85] address lint issue for httpservicefake --- cns/fakes/cnsfake.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cns/fakes/cnsfake.go b/cns/fakes/cnsfake.go index 07618b01e4..3f143e06bf 100644 --- a/cns/fakes/cnsfake.go +++ b/cns/fakes/cnsfake.go @@ -276,3 +276,5 @@ func (fake *HTTPServiceFake) Init(*common.ServiceConfig) error { } func (fake *HTTPServiceFake) Stop() {} + +func (fake *HTTPServiceFake) AttachMultitenantMiddleware(cns.MultitenantMiddleware) {} From f2359ac80cca5f2eea1fba6d8e58283685f151bb Mon Sep 17 00:00:00 2001 From: Quang Nguyen Date: Mon, 21 Aug 2023 20:21:10 +0000 Subject: [PATCH 10/85] getting pod info in validator --- cns/middlewares/multitenant.go | 33 ++++++++++++++++++++++++++------- cns/service/main.go | 8 ++++++-- 2 files changed, 32 insertions(+), 9 deletions(-) diff --git a/cns/middlewares/multitenant.go b/cns/middlewares/multitenant.go index 94ad1956c0..fff83f75e7 100644 --- a/cns/middlewares/multitenant.go +++ b/cns/middlewares/multitenant.go @@ -1,8 +1,14 @@ package middlewares import ( + "context" + "fmt" + "github.com/Azure/azure-container-networking/cns" + "github.com/Azure/azure-container-networking/cns/configuration" "github.com/Azure/azure-container-networking/cns/types" + v1 "k8s.io/api/core/v1" + k8types "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -24,14 +30,27 @@ func (m *MultitenantMiddleware) Validator() cns.IPConfigValidator { return m.validateMultitenantIPConfigsRequest } -// validateMultitenantIPConfigsRequest validate whether the request is for a multitenant pod +// validateMultitenantIPConfigsRequest validates if pod is multitenant // nolint -func (m *MultitenantMiddleware) validateMultitenantIPConfigsRequest(ipConfigsRequest *cns.IPConfigsRequest) (respCode types.ResponseCode, message string) { - /** - TODO: - - Check if pod is multitenant, enrich the request with the multitenant flag - **/ - ipConfigsRequest.Multitenant = true +func (m *MultitenantMiddleware) validateMultitenantIPConfigsRequest(req *cns.IPConfigsRequest) (respCode types.ResponseCode, message string) { + // Retrieve the pod from the cluster + podInfo, err := cns.UnmarshalPodInfo(req.OrchestratorContext) + if err != nil { + errBuf := fmt.Sprintf("unmarshalling pod info from ipconfigs request %v failed with error %v", req, err) + return types.UnexpectedError, errBuf + } + podNamespacedName := k8types.NamespacedName{Namespace: podInfo.Namespace(), Name: podInfo.Name()} + pod := v1.Pod{} + err = m.cli.Get(context.TODO(), podNamespacedName, &pod) + if err != nil { + errBuf := fmt.Sprintf("failed to get pod %v with error %v", podNamespacedName, err) + return types.UnexpectedError, errBuf + } + + // check the pod labels for Swift V2, enrich the request with the multitenant flag. TBD on the label + if _, ok := pod.Labels[configuration.LabelSwiftV2]; ok { + req.Multitenant = true + } return types.Success, "" } diff --git a/cns/service/main.go b/cns/service/main.go index 81f52d54d5..a4f15fc77f 100644 --- a/cns/service/main.go +++ b/cns/service/main.go @@ -55,7 +55,8 @@ import ( "github.com/avast/retry-go/v3" "github.com/pkg/errors" "go.uber.org/zap" - v1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/apps/v1" + v1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" @@ -1191,6 +1192,9 @@ func InitializeCRDState(ctx context.Context, httpRestService cns.HTTPService, cn &v1alpha.NodeNetworkConfig{}: { Field: fields.SelectorFromSet(fields.Set{"metadata.name": nodeName}), }, + &v1.Pod{}: { + Field: fields.SelectorFromSet(fields.Set{"spec.nodeName": nodeName}), + }, }, }) @@ -1201,7 +1205,7 @@ func InitializeCRDState(ctx context.Context, httpRestService cns.HTTPService, cn if err = v1alpha1.AddToScheme(crdSchemes); err != nil { return errors.Wrap(err, "failed to add clustersubnetstate/v1alpha1 to scheme") } - if err = v1.AddToScheme(crdSchemes); err != nil { + if err = corev1.AddToScheme(crdSchemes); err != nil { return errors.Wrap(err, "failed to add core/v1 to scheme") } From fa2cac529b8dad664b1811044b4c8f1223c2f72a Mon Sep 17 00:00:00 2001 From: Quang Nguyen Date: Mon, 21 Aug 2023 20:38:18 +0000 Subject: [PATCH 11/85] linter issue --- cns/service/main.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/cns/service/main.go b/cns/service/main.go index a4f15fc77f..5d3eca3272 100644 --- a/cns/service/main.go +++ b/cns/service/main.go @@ -55,8 +55,7 @@ import ( "github.com/avast/retry-go/v3" "github.com/pkg/errors" "go.uber.org/zap" - corev1 "k8s.io/api/apps/v1" - v1 "k8s.io/api/core/v1" + corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" @@ -1192,7 +1191,7 @@ func InitializeCRDState(ctx context.Context, httpRestService cns.HTTPService, cn &v1alpha.NodeNetworkConfig{}: { Field: fields.SelectorFromSet(fields.Set{"metadata.name": nodeName}), }, - &v1.Pod{}: { + &corev1.Pod{}: { // nolint: typecheck Field: fields.SelectorFromSet(fields.Set{"spec.nodeName": nodeName}), }, }, From cc91071472e74bf99d6087c40b5f45f5a02fa090 Mon Sep 17 00:00:00 2001 From: Quang Nguyen Date: Wed, 23 Aug 2023 16:58:46 +0000 Subject: [PATCH 12/85] update network container contract --- cns/NetworkContainerContract.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cns/NetworkContainerContract.go b/cns/NetworkContainerContract.go index 36b6d3768a..6bf9ced2ca 100644 --- a/cns/NetworkContainerContract.go +++ b/cns/NetworkContainerContract.go @@ -414,6 +414,8 @@ type PodIpInfo struct { HostPrimaryIPInfo HostIPInfo AddressType string // multitenant or default, maybe make this an enum later MACAddress string + IsDefaultInterface bool + Routes []Route } type HostIPInfo struct { From 0674e06115917710da5c101caa513ecb510b657c Mon Sep 17 00:00:00 2001 From: Quang Nguyen Date: Fri, 25 Aug 2023 16:24:26 +0000 Subject: [PATCH 13/85] renaming --- cns/api.go | 4 ++-- cns/fakes/cnsfake.go | 2 +- cns/middlewares/{multitenant.go => SWIFTv2.go} | 12 ++++++------ cns/restserver/restserver.go | 4 ++-- cns/service/main.go | 4 ++-- 5 files changed, 13 insertions(+), 13 deletions(-) rename cns/middlewares/{multitenant.go => SWIFTv2.go} (78%) diff --git a/cns/api.go b/cns/api.go index 8595d617ee..caea20e141 100644 --- a/cns/api.go +++ b/cns/api.go @@ -48,11 +48,11 @@ type HTTPService interface { GetPendingReleaseIPConfigs() []IPConfigurationStatus GetPodIPConfigState() map[string]IPConfigurationStatus MarkIPAsPendingRelease(numberToMark int) (map[string]IPConfigurationStatus, error) - AttachMultitenantMiddleware(middleware MultitenantMiddleware) + AttachSWIFTv2Middleware(middleware SWIFTv2Middleware) } // Middleware interface for testing later on -type MultitenantMiddleware interface { +type SWIFTv2Middleware interface { Validator() IPConfigValidator GetMultitenantIPConfig(podInfo PodInfo) (*PodIpInfo, error) } diff --git a/cns/fakes/cnsfake.go b/cns/fakes/cnsfake.go index 3f143e06bf..aed304f976 100644 --- a/cns/fakes/cnsfake.go +++ b/cns/fakes/cnsfake.go @@ -277,4 +277,4 @@ func (fake *HTTPServiceFake) Init(*common.ServiceConfig) error { func (fake *HTTPServiceFake) Stop() {} -func (fake *HTTPServiceFake) AttachMultitenantMiddleware(cns.MultitenantMiddleware) {} +func (fake *HTTPServiceFake) AttachSWIFTv2Middleware(cns.SWIFTv2Middleware) {} diff --git a/cns/middlewares/multitenant.go b/cns/middlewares/SWIFTv2.go similarity index 78% rename from cns/middlewares/multitenant.go rename to cns/middlewares/SWIFTv2.go index fff83f75e7..1a9cbb054d 100644 --- a/cns/middlewares/multitenant.go +++ b/cns/middlewares/SWIFTv2.go @@ -12,27 +12,27 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) -type MultitenantMiddleware struct { +type SWIFTv2Middleware struct { // TODO: implement // need cached scoped client for pods // need client for MTPNC CRD for x-ref pods cli client.Client } -func NewMultitenantMiddleware(cli client.Client) *MultitenantMiddleware { - return &MultitenantMiddleware{ +func NewSWIFTv2Middleware(cli client.Client) *SWIFTv2Middleware { + return &SWIFTv2Middleware{ cli: cli, } } // Return the validator function for the middleware -func (m *MultitenantMiddleware) Validator() cns.IPConfigValidator { +func (m *SWIFTv2Middleware) Validator() cns.IPConfigValidator { return m.validateMultitenantIPConfigsRequest } // validateMultitenantIPConfigsRequest validates if pod is multitenant // nolint -func (m *MultitenantMiddleware) validateMultitenantIPConfigsRequest(req *cns.IPConfigsRequest) (respCode types.ResponseCode, message string) { +func (m *SWIFTv2Middleware) validateMultitenantIPConfigsRequest(req *cns.IPConfigsRequest) (respCode types.ResponseCode, message string) { // Retrieve the pod from the cluster podInfo, err := cns.UnmarshalPodInfo(req.OrchestratorContext) if err != nil { @@ -56,7 +56,7 @@ func (m *MultitenantMiddleware) validateMultitenantIPConfigsRequest(req *cns.IPC // nolint // GetMultitenantIPConfig returns the IP config for a multitenant pod from the MTPNC CRD -func (m *MultitenantMiddleware) GetMultitenantIPConfig(podInfo cns.PodInfo) (*cns.PodIpInfo, error) { +func (m *SWIFTv2Middleware) GetMultitenantIPConfig(podInfo cns.PodInfo) (*cns.PodIpInfo, error) { /** TODO: - Check if the MTPNC CRD exists for the pod, if not, return error diff --git a/cns/restserver/restserver.go b/cns/restserver/restserver.go index ecef0123da..0c8fdcdf2c 100644 --- a/cns/restserver/restserver.go +++ b/cns/restserver/restserver.go @@ -73,7 +73,7 @@ type HTTPRestService struct { cniConflistGenerator CNIConflistGenerator generateCNIConflistOnce sync.Once ipConfigsValidators []cns.IPConfigValidator - MultitenantMiddleware cns.MultitenantMiddleware + MultitenantMiddleware cns.SWIFTv2Middleware } type CNIConflistGenerator interface { @@ -353,7 +353,7 @@ func (service *HTTPRestService) MustGenerateCNIConflistOnce() { }) } -func (service *HTTPRestService) AttachMultitenantMiddleware(middleware cns.MultitenantMiddleware) { +func (service *HTTPRestService) AttachSWIFTv2Middleware(middleware cns.SWIFTv2Middleware) { service.MultitenantMiddleware = middleware // add multitenant ipconfig validator function service.ipConfigsValidators = append(service.ipConfigsValidators, middleware.Validator()) diff --git a/cns/service/main.go b/cns/service/main.go index 2dbd5f23f2..e1a9eec1bc 100644 --- a/cns/service/main.go +++ b/cns/service/main.go @@ -1261,8 +1261,8 @@ func InitializeCRDState(ctx context.Context, httpRestService cns.HTTPService, cn cnsconfig.EnableSwiftV2 = true // TODO(rbtr): create the NodeInfo for Swift V2 // if SWIFT v2 is enabled on CNS, attach multitenant middleware to rest service - multitenantMiddleware := middlewares.NewMultitenantMiddleware(manager.GetClient()) - httpRestService.AttachMultitenantMiddleware(multitenantMiddleware) + swiftV2Middleware := middlewares.NewSWIFTv2Middleware(manager.GetClient()) + httpRestService.AttachSWIFTv2Middleware(swiftV2Middleware) } From fd03f7cd8518e59182586982b51dbe576986e601 Mon Sep 17 00:00:00 2001 From: Quang Nguyen Date: Tue, 29 Aug 2023 16:05:23 +0000 Subject: [PATCH 14/85] mtpnc changes --- cns/NetworkContainerContract.go | 6 ++++++ cns/api.go | 2 +- cns/middlewares/SWIFTv2.go | 17 ++++++++++++++++- cns/restserver/ipam.go | 2 +- 4 files changed, 24 insertions(+), 3 deletions(-) diff --git a/cns/NetworkContainerContract.go b/cns/NetworkContainerContract.go index 6bf9ced2ca..f404329845 100644 --- a/cns/NetworkContainerContract.go +++ b/cns/NetworkContainerContract.go @@ -77,6 +77,12 @@ const ( MultiTenantCRD = "MultiTenantCRD" ) +// Address Types +const ( + Default = "Default" + Multitenant = "Multitenant" +) + // CreateNetworkContainerRequest specifies request to create a network container or network isolation boundary. type CreateNetworkContainerRequest struct { HostPrimaryIP string diff --git a/cns/api.go b/cns/api.go index caea20e141..ce794b48e3 100644 --- a/cns/api.go +++ b/cns/api.go @@ -54,7 +54,7 @@ type HTTPService interface { // Middleware interface for testing later on type SWIFTv2Middleware interface { Validator() IPConfigValidator - GetMultitenantIPConfig(podInfo PodInfo) (*PodIpInfo, error) + GetSWIFTv2IPConfig(podInfo PodInfo) (*PodIpInfo, error) } type IPConfigValidator func(ipConfigsRequest *IPConfigsRequest) (types.ResponseCode, string) diff --git a/cns/middlewares/SWIFTv2.go b/cns/middlewares/SWIFTv2.go index 1a9cbb054d..9b0264ffb8 100644 --- a/cns/middlewares/SWIFTv2.go +++ b/cns/middlewares/SWIFTv2.go @@ -7,6 +7,7 @@ import ( "github.com/Azure/azure-container-networking/cns" "github.com/Azure/azure-container-networking/cns/configuration" "github.com/Azure/azure-container-networking/cns/types" + "github.com/Azure/azure-container-networking/crd/multitenancy/api/v1alpha1" v1 "k8s.io/api/core/v1" k8types "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" @@ -56,10 +57,24 @@ func (m *SWIFTv2Middleware) validateMultitenantIPConfigsRequest(req *cns.IPConfi // nolint // GetMultitenantIPConfig returns the IP config for a multitenant pod from the MTPNC CRD -func (m *SWIFTv2Middleware) GetMultitenantIPConfig(podInfo cns.PodInfo) (*cns.PodIpInfo, error) { +func (m *SWIFTv2Middleware) GetSWIFTv2IPConfig(podInfo cns.PodInfo) (*cns.PodIpInfo, error) { /** TODO: - Check if the MTPNC CRD exists for the pod, if not, return error **/ + mtpnc := v1alpha1.MultitenantPodNetworkConfig{} + mtpncNamespacedName := k8types.NamespacedName{Namespace: podInfo.Namespace(), Name: podInfo.Name()} + err := m.cli.Get(context.Background(), mtpncNamespacedName, &mtpnc) + if err != nil { + return nil, fmt.Errorf("failed to get mtpnc %v with error %v", mtpncNamespacedName, err) + } + podIpInfo := cns.PodIpInfo{} + podIpInfo.PodIPConfig = cns.IPSubnet{ + IPAddress: mtpnc.Status.PrimaryIP, + } + podIpInfo.MACAddress = mtpnc.Status.MacAddress + podIpInfo.AddressType = cns.Multitenant + podIpInfo.IsDefaultInterface = true // should not assume this but for SWIFT v2, this is true + return nil, nil } diff --git a/cns/restserver/ipam.go b/cns/restserver/ipam.go index e34997e5f0..54946bacaa 100644 --- a/cns/restserver/ipam.go +++ b/cns/restserver/ipam.go @@ -50,7 +50,7 @@ func (service *HTTPRestService) requestIPConfigHandlerHelper(ipconfigsRequest cn }, errors.New("request is for multitenant pod but multitenant middleware is nil") } // If pod is multitenant and we failed to grab its MTPNC IP config, return error immediately - podIPInfo, err := service.MultitenantMiddleware.GetMultitenantIPConfig(podInfo) + podIPInfo, err := service.MultitenantMiddleware.GetSWIFTv2IPConfig(podInfo) if err != nil { return &cns.IPConfigsResponse{ Response: cns.Response{ From 1e7c99dd25e5b2d67a1e5386a3545c828adaa0c7 Mon Sep 17 00:00:00 2001 From: Quang Nguyen Date: Tue, 29 Aug 2023 16:10:47 +0000 Subject: [PATCH 15/85] rebase --- .../{buildinfo/buildinfo.go => buildversion/buildversion.go} | 4 ++-- azure-ipam/ipam.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) rename azure-ipam/internal/{buildinfo/buildinfo.go => buildversion/buildversion.go} (74%) diff --git a/azure-ipam/internal/buildinfo/buildinfo.go b/azure-ipam/internal/buildversion/buildversion.go similarity index 74% rename from azure-ipam/internal/buildinfo/buildinfo.go rename to azure-ipam/internal/buildversion/buildversion.go index e26b8d2f8e..e25cccd172 100644 --- a/azure-ipam/internal/buildinfo/buildinfo.go +++ b/azure-ipam/internal/buildversion/buildversion.go @@ -1,6 +1,6 @@ -package buildinfo +package buildversion // this will be populate by the Go compiler via // the -ldflags, which insert dynamic information // into the binary at build time -var Version string +var BuildVersion string diff --git a/azure-ipam/ipam.go b/azure-ipam/ipam.go index b3803a3c7d..da7c23ac7e 100644 --- a/azure-ipam/ipam.go +++ b/azure-ipam/ipam.go @@ -6,7 +6,7 @@ import ( "io" "net" - "github.com/Azure/azure-container-networking/azure-ipam/internal/buildinfo" + "github.com/Azure/azure-container-networking/azure-ipam/internal/buildversion" "github.com/Azure/azure-container-networking/azure-ipam/ipconfig" "github.com/Azure/azure-container-networking/cns" cniSkel "github.com/containernetworking/cni/pkg/skel" @@ -36,7 +36,7 @@ type cnsClient interface { func NewPlugin(logger *zap.Logger, c cnsClient, out io.Writer) (*IPAMPlugin, error) { plugin := &IPAMPlugin{ Name: pluginName, - Version: buildinfo.Version, + Version: buildversion.BuildVersion, logger: logger, out: out, cnsClient: c, From 6e71052c48645a622744827d624329517d7c163e Mon Sep 17 00:00:00 2001 From: Quang Nguyen Date: Mon, 31 Jul 2023 22:09:59 +0000 Subject: [PATCH 16/85] revert file and var naming, add correct path to makefile --- .../{buildversion/buildversion.go => buildinfo/buildinfo.go} | 4 ++-- azure-ipam/ipam.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) rename azure-ipam/internal/{buildversion/buildversion.go => buildinfo/buildinfo.go} (74%) diff --git a/azure-ipam/internal/buildversion/buildversion.go b/azure-ipam/internal/buildinfo/buildinfo.go similarity index 74% rename from azure-ipam/internal/buildversion/buildversion.go rename to azure-ipam/internal/buildinfo/buildinfo.go index e25cccd172..e26b8d2f8e 100644 --- a/azure-ipam/internal/buildversion/buildversion.go +++ b/azure-ipam/internal/buildinfo/buildinfo.go @@ -1,6 +1,6 @@ -package buildversion +package buildinfo // this will be populate by the Go compiler via // the -ldflags, which insert dynamic information // into the binary at build time -var BuildVersion string +var Version string diff --git a/azure-ipam/ipam.go b/azure-ipam/ipam.go index da7c23ac7e..b3803a3c7d 100644 --- a/azure-ipam/ipam.go +++ b/azure-ipam/ipam.go @@ -6,7 +6,7 @@ import ( "io" "net" - "github.com/Azure/azure-container-networking/azure-ipam/internal/buildversion" + "github.com/Azure/azure-container-networking/azure-ipam/internal/buildinfo" "github.com/Azure/azure-container-networking/azure-ipam/ipconfig" "github.com/Azure/azure-container-networking/cns" cniSkel "github.com/containernetworking/cni/pkg/skel" @@ -36,7 +36,7 @@ type cnsClient interface { func NewPlugin(logger *zap.Logger, c cnsClient, out io.Writer) (*IPAMPlugin, error) { plugin := &IPAMPlugin{ Name: pluginName, - Version: buildversion.BuildVersion, + Version: buildinfo.Version, logger: logger, out: out, cnsClient: c, From ad1bf077859d302cc5b473ac54285ce1300f0a2f Mon Sep 17 00:00:00 2001 From: Quang Nguyen Date: Tue, 29 Aug 2023 20:00:44 +0000 Subject: [PATCH 17/85] add default route --- cns/middlewares/SWIFTv2.go | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/cns/middlewares/SWIFTv2.go b/cns/middlewares/SWIFTv2.go index 9b0264ffb8..6cbd17a46d 100644 --- a/cns/middlewares/SWIFTv2.go +++ b/cns/middlewares/SWIFTv2.go @@ -2,6 +2,7 @@ package middlewares import ( "context" + "errors" "fmt" "github.com/Azure/azure-container-networking/cns" @@ -58,23 +59,36 @@ func (m *SWIFTv2Middleware) validateMultitenantIPConfigsRequest(req *cns.IPConfi // nolint // GetMultitenantIPConfig returns the IP config for a multitenant pod from the MTPNC CRD func (m *SWIFTv2Middleware) GetSWIFTv2IPConfig(podInfo cns.PodInfo) (*cns.PodIpInfo, error) { - /** - TODO: - - Check if the MTPNC CRD exists for the pod, if not, return error - **/ + + // Check if the MTPNC CRD exists for the pod, if not, return error mtpnc := v1alpha1.MultitenantPodNetworkConfig{} mtpncNamespacedName := k8types.NamespacedName{Namespace: podInfo.Namespace(), Name: podInfo.Name()} err := m.cli.Get(context.Background(), mtpncNamespacedName, &mtpnc) if err != nil { return nil, fmt.Errorf("failed to get mtpnc %v with error %v", mtpncNamespacedName, err) } + + // Check if the MTPNC CRD is ready. If one of the fields is empty, return error + if mtpnc.Status.PrimaryIP == "" || mtpnc.Status.MacAddress == "" || mtpnc.Status.NCID == "" || mtpnc.Status.GatewayIP == "" { + return nil, errors.New("one or more of mtpnc's status fields is empty. mtpnc is not ready") + } podIpInfo := cns.PodIpInfo{} podIpInfo.PodIPConfig = cns.IPSubnet{ IPAddress: mtpnc.Status.PrimaryIP, } podIpInfo.MACAddress = mtpnc.Status.MacAddress podIpInfo.AddressType = cns.Multitenant - podIpInfo.IsDefaultInterface = true // should not assume this but for SWIFT v2, this is true + podIpInfo.IsDefaultInterface = true + + /** + TODO: add routes, find out where underlay pod/service cidr, + underlay pod gateway, pod cidr and pod ip are stored. + **/ + defaultRoute := cns.Route{ + IPAddress: mtpnc.Status.PrimaryIP, + GatewayIPAddress: mtpnc.Status.GatewayIP, + InterfaceToUse: "eth1"} + podIpInfo.Routes = []cns.Route{defaultRoute} return nil, nil } From e26f7c3dc378867a64c8d4025bf117a8e5d4b3d3 Mon Sep 17 00:00:00 2001 From: Quang Nguyen Date: Fri, 8 Sep 2023 18:11:47 +0000 Subject: [PATCH 18/85] add unit tests --- cns/configuration/env.go | 12 ++- cns/middlewares/SWIFTv2.go | 41 +++++---- cns/middlewares/SWIFTv2_test.go | 86 +++++++++++++++++++ cns/middlewares/mock/mockClient.go | 127 ++++++++++++++++++++++++++++ cns/middlewares/mock/mockSWIFTv2.go | 97 +++++++++++++++++++++ cns/restserver/ipam.go | 10 --- 6 files changed, 346 insertions(+), 27 deletions(-) create mode 100644 cns/middlewares/SWIFTv2_test.go create mode 100644 cns/middlewares/mock/mockClient.go create mode 100644 cns/middlewares/mock/mockSWIFTv2.go diff --git a/cns/configuration/env.go b/cns/configuration/env.go index 5713235df0..a6a77f66f5 100644 --- a/cns/configuration/env.go +++ b/cns/configuration/env.go @@ -12,7 +12,9 @@ const ( // EnvNodeIP is the IP of the node running this CNS binary EnvNodeIP = "NODE_IP" // LabelSwiftV2 is the Node label for Swift V2 - LabelSwiftV2 = "kubernetes.azure.com/podnetwork-multi-tenancy" + LabelSwiftV2 = "kubernetes.azure.com/podnetwork-multi-tenancy" + EnvPodCIDR = "PodCIDR" + EnvServiceCIDR = "ServiceCIDR" ) // ErrNodeNameUnset indicates the the $EnvNodeName variable is unset in the environment. @@ -31,3 +33,11 @@ func NodeName() (string, error) { func NodeIP() string { return os.Getenv(EnvNodeIP) } + +func PodCIDR() string { + return os.Getenv(EnvPodCIDR) +} + +func ServiceCIDR() string { + return os.Getenv(EnvServiceCIDR) +} diff --git a/cns/middlewares/SWIFTv2.go b/cns/middlewares/SWIFTv2.go index 6cbd17a46d..7da957f9c3 100644 --- a/cns/middlewares/SWIFTv2.go +++ b/cns/middlewares/SWIFTv2.go @@ -14,6 +14,8 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) +var ErrMTPNCNotReady = errors.New("mtpnc is not ready") + type SWIFTv2Middleware struct { // TODO: implement // need cached scoped client for pods @@ -56,39 +58,46 @@ func (m *SWIFTv2Middleware) validateMultitenantIPConfigsRequest(req *cns.IPConfi return types.Success, "" } -// nolint // GetMultitenantIPConfig returns the IP config for a multitenant pod from the MTPNC CRD func (m *SWIFTv2Middleware) GetSWIFTv2IPConfig(podInfo cns.PodInfo) (*cns.PodIpInfo, error) { - // Check if the MTPNC CRD exists for the pod, if not, return error mtpnc := v1alpha1.MultitenantPodNetworkConfig{} mtpncNamespacedName := k8types.NamespacedName{Namespace: podInfo.Namespace(), Name: podInfo.Name()} err := m.cli.Get(context.Background(), mtpncNamespacedName, &mtpnc) if err != nil { - return nil, fmt.Errorf("failed to get mtpnc %v with error %v", mtpncNamespacedName, err) + return nil, fmt.Errorf("failed to get pod's mtpnc from cache : %w", err) } // Check if the MTPNC CRD is ready. If one of the fields is empty, return error if mtpnc.Status.PrimaryIP == "" || mtpnc.Status.MacAddress == "" || mtpnc.Status.NCID == "" || mtpnc.Status.GatewayIP == "" { - return nil, errors.New("one or more of mtpnc's status fields is empty. mtpnc is not ready") + return nil, ErrMTPNCNotReady } - podIpInfo := cns.PodIpInfo{} - podIpInfo.PodIPConfig = cns.IPSubnet{ + podIPInfo := cns.PodIpInfo{} + podIPInfo.PodIPConfig = cns.IPSubnet{ IPAddress: mtpnc.Status.PrimaryIP, } - podIpInfo.MACAddress = mtpnc.Status.MacAddress - podIpInfo.AddressType = cns.Multitenant - podIpInfo.IsDefaultInterface = true + podIPInfo.MACAddress = mtpnc.Status.MacAddress + podIPInfo.AddressType = cns.Multitenant + podIPInfo.IsDefaultInterface = true - /** - TODO: add routes, find out where underlay pod/service cidr, - underlay pod gateway, pod cidr and pod ip are stored. - **/ defaultRoute := cns.Route{ IPAddress: mtpnc.Status.PrimaryIP, GatewayIPAddress: mtpnc.Status.GatewayIP, - InterfaceToUse: "eth1"} - podIpInfo.Routes = []cns.Route{defaultRoute} + InterfaceToUse: "eth1", + } + + podCIDRRoute := cns.Route{ + IPAddress: configuration.PodCIDR(), + GatewayIPAddress: mtpnc.Status.GatewayIP, + InterfaceToUse: "eth0", + } + + serviceCIDRRoute := cns.Route{ + IPAddress: configuration.ServiceCIDR(), + GatewayIPAddress: mtpnc.Status.GatewayIP, + InterfaceToUse: "eth0", + } + podIPInfo.Routes = []cns.Route{defaultRoute, podCIDRRoute, serviceCIDRRoute} - return nil, nil + return &podIPInfo, nil } diff --git a/cns/middlewares/SWIFTv2_test.go b/cns/middlewares/SWIFTv2_test.go new file mode 100644 index 0000000000..25515b4bf8 --- /dev/null +++ b/cns/middlewares/SWIFTv2_test.go @@ -0,0 +1,86 @@ +package middlewares + +import ( + "testing" + + "github.com/Azure/azure-container-networking/cns" + mock "github.com/Azure/azure-container-networking/cns/middlewares/mock" + "github.com/Azure/azure-container-networking/cns/types" + "gotest.tools/v3/assert" +) + +var ( + testPod1GUID = "898fb8f1-f93e-4c96-9c31-6b89098949a3" + testPod1Info = cns.NewPodInfo("898fb8-eth0", testPod1GUID, "testpod1", "testpod1namespace") + + testPod2GUID = "b21e1ee1-fb7e-4e6d-8c68-22ee5049944e" + testPod2Info = cns.NewPodInfo("b21e1e-eth0", testPod2GUID, "testpod2", "testpod2namespace") + + testPod3GUID = "718e04ac-5a13-4dce-84b3-040accaa9b41" + testPod3Info = cns.NewPodInfo("718e04-eth0", testPod3GUID, "testpod3", "testpod3namespace") +) + +func TestValidateMultitenantIPConfigsRequestSuccess(t *testing.T) { + middleware := NewSWIFTv2Middleware(mock.NewMockClient()) + + validator := middleware.Validator() + + happyReq := &cns.IPConfigsRequest{ + PodInterfaceID: testPod1Info.InterfaceID(), + InfraContainerID: testPod1Info.InfraContainerID(), + } + b, _ := testPod1Info.OrchestratorContext() + happyReq.OrchestratorContext = b + happyReq.Multitenant = false + + respCode, err := validator(happyReq) + assert.Equal(t, err, "") + assert.Equal(t, respCode, types.Success) + assert.Equal(t, happyReq.Multitenant, true) +} + +func TestValidateMultitenantIPConfigsRequestFailure(t *testing.T) { + middleware := NewSWIFTv2Middleware(mock.NewMockClient()) + + validator := middleware.Validator() + + // Fail to unmarshal pod info test + failReq := &cns.IPConfigsRequest{ + PodInterfaceID: testPod1Info.InterfaceID(), + InfraContainerID: testPod1Info.InfraContainerID(), + } + failReq.OrchestratorContext = []byte("invalid") + respCode, _ := validator(failReq) + assert.Equal(t, respCode, types.UnexpectedError) + + // Pod doesn't exist in cache test + failReq = &cns.IPConfigsRequest{ + PodInterfaceID: testPod2Info.InterfaceID(), + InfraContainerID: testPod2Info.InfraContainerID(), + } + b, _ := testPod2Info.OrchestratorContext() + failReq.OrchestratorContext = b + respCode, _ = validator(failReq) + assert.Equal(t, respCode, types.UnexpectedError) +} + +func TestGetSWIFTv2IPConfigSuccess(t *testing.T) { + middleware := NewSWIFTv2Middleware(mock.NewMockClient()) + + ipInfo, err := middleware.GetSWIFTv2IPConfig(testPod1Info) + assert.Equal(t, err, nil) + assert.Equal(t, ipInfo.AddressType, cns.Multitenant) + assert.Equal(t, ipInfo.IsDefaultInterface, true) +} + +func TestGetSWIFTv2IPConfigFailure(t *testing.T) { + middleware := NewSWIFTv2Middleware(mock.NewMockClient()) + + // Pod's MTPNC doesn't exist in cache test + _, err := middleware.GetSWIFTv2IPConfig(testPod2Info) + assert.Error(t, err, "failed to get pod's mtpnc from cache : mtpnc not found") + + // Pod's MTPNC is not ready test + _, err = middleware.GetSWIFTv2IPConfig(testPod3Info) + assert.Error(t, err, ErrMTPNCNotReady.Error()) +} diff --git a/cns/middlewares/mock/mockClient.go b/cns/middlewares/mock/mockClient.go new file mode 100644 index 0000000000..c99ef8f828 --- /dev/null +++ b/cns/middlewares/mock/mockClient.go @@ -0,0 +1,127 @@ +package middlewares + +import ( + "context" + "errors" + + "github.com/Azure/azure-container-networking/cns/configuration" + "github.com/Azure/azure-container-networking/crd/multitenancy/api/v1alpha1" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +var ( + errPodNotFound = errors.New("pod not found") + errMTPNCNotFound = errors.New("mtpnc not found") +) + +// MockClient implements the client.Client interface for testing. We only care about Get. +type MockClient struct { + mtPodCache map[string]*v1.Pod + mtpncCache map[string]*v1alpha1.MultitenantPodNetworkConfig +} + +// NewMockClient returns a new MockClient. +func NewMockClient() *MockClient { + testPod1 := v1.Pod{} + testPod1.Labels = make(map[string]string) + testPod1.Labels[configuration.LabelSwiftV2] = "true" + + testMTPNC1 := v1alpha1.MultitenantPodNetworkConfig{} + testMTPNC1.Status.PrimaryIP = "192.168.0.1" + testMTPNC1.Status.MacAddress = "00:00:00:00:00:00" + testMTPNC1.Status.GatewayIP = "10.0.0.1" + testMTPNC1.Status.NCID = "testncid" + + testMTPNC3 := v1alpha1.MultitenantPodNetworkConfig{} + + return &MockClient{ + mtPodCache: map[string]*v1.Pod{"testpod1namespace/testpod1": &testPod1}, + mtpncCache: map[string]*v1alpha1.MultitenantPodNetworkConfig{ + "testpod1namespace/testpod1": &testMTPNC1, + "testpod3namespace/testpod3": &testMTPNC3, + }, + } +} + +// Get implements client.Client.Get. +func (c *MockClient) Get(_ context.Context, key client.ObjectKey, obj client.Object, _ ...client.GetOption) error { + switch o := obj.(type) { + case *v1.Pod: + if pod, ok := c.mtPodCache[key.String()]; ok { + *o = *pod + } else { + return errPodNotFound + } + case *v1alpha1.MultitenantPodNetworkConfig: + if mtpnc, ok := c.mtpncCache[key.String()]; ok { + *o = *mtpnc + } else { + return errMTPNCNotFound + } + } + return nil +} + +// List implements client.Client.List. +func (c *MockClient) List(_ context.Context, _ client.ObjectList, _ ...client.ListOption) error { + return nil +} + +// Create implements client.Client.Create. +func (c *MockClient) Create(_ context.Context, _ client.Object, _ ...client.CreateOption) error { + return nil +} + +// Delete implements client.Client.Delete. +func (c *MockClient) Delete(_ context.Context, _ client.Object, _ ...client.DeleteOption) error { + return nil +} + +// Update implements client.Client.Update. +func (c *MockClient) Update(_ context.Context, _ client.Object, _ ...client.UpdateOption) error { + return nil +} + +// Patch implements client.Client.Patch. +func (c *MockClient) Patch(_ context.Context, _ client.Object, _ client.Patch, _ ...client.PatchOption) error { + return nil +} + +// DeleteAllOf implements client.Client.DeleteAllOf. +func (c *MockClient) DeleteAllOf(_ context.Context, _ client.Object, _ ...client.DeleteAllOfOption) error { + return nil +} + +// Status implements client.StatusClient. +func (c *MockClient) Status() client.StatusWriter { + return nil +} + +// RESTMapper implements client.Client. +func (c *MockClient) RESTMapper() meta.RESTMapper { + return nil +} + +// Scheme implements client.Client. +func (c *MockClient) Scheme() *runtime.Scheme { + return nil +} + +// GroupVersionKindFor implements client.Client. +func (c *MockClient) GroupVersionKindFor(_ runtime.Object) (schema.GroupVersionKind, error) { + return schema.GroupVersionKind{}, nil +} + +// IsObjectNamespaced implements client.Client. +func (c *MockClient) IsObjectNamespaced(_ runtime.Object) (bool, error) { + return false, nil +} + +// SubResource implements client.Client. +func (c *MockClient) SubResource(_ string) client.SubResourceClient { + return nil +} diff --git a/cns/middlewares/mock/mockSWIFTv2.go b/cns/middlewares/mock/mockSWIFTv2.go new file mode 100644 index 0000000000..4d0be95df1 --- /dev/null +++ b/cns/middlewares/mock/mockSWIFTv2.go @@ -0,0 +1,97 @@ +package middlewares + +import ( + "errors" + "fmt" + + "github.com/Azure/azure-container-networking/cns" + "github.com/Azure/azure-container-networking/cns/configuration" + "github.com/Azure/azure-container-networking/cns/types" + "github.com/Azure/azure-container-networking/crd/multitenancy/api/v1alpha1" + v1 "k8s.io/api/core/v1" + k8types "k8s.io/apimachinery/pkg/types" +) + +var ( + errMTPNCNotReady = errors.New("mtpnc is not ready") + errFailedToGetPod = errors.New("failed to get pod") +) + +type MockSWIFTv2Middleware struct { + mtPodState map[string]*v1.Pod + mtpncState map[string]*v1alpha1.MultitenantPodNetworkConfig +} + +func NewMockSWIFTv2Middleware() *MockSWIFTv2Middleware { + testPod1 := v1.Pod{} + testPod1.Labels = make(map[string]string) + testPod1.Labels[configuration.LabelSwiftV2] = "true" + + testMTPNC1 := v1alpha1.MultitenantPodNetworkConfig{} + testMTPNC1.Status.PrimaryIP = "192.168.0.1" + testMTPNC1.Status.MacAddress = "00:00:00:00:00:00" + + return &MockSWIFTv2Middleware{ + mtPodState: map[string]*v1.Pod{"testpod1namespace/testpod1": &testPod1}, + mtpncState: map[string]*v1alpha1.MultitenantPodNetworkConfig{"testpod1namespace/testpod1": &testMTPNC1}, + } +} + +// Return the validator function for the middleware +func (m *MockSWIFTv2Middleware) Validator() cns.IPConfigValidator { + return m.validateMultitenantIPConfigsRequest +} + +// validateMultitenantIPConfigsRequest validates if pod is multitenant +// nolint +func (m *MockSWIFTv2Middleware) validateMultitenantIPConfigsRequest(req *cns.IPConfigsRequest) (respCode types.ResponseCode, message string) { + // Retrieve the pod from the cluster + podInfo, err := cns.UnmarshalPodInfo(req.OrchestratorContext) + if err != nil { + errBuf := fmt.Sprintf("unmarshalling pod info from ipconfigs request %v failed with error %v", req, err) + return types.UnexpectedError, errBuf + } + podNamespacedName := k8types.NamespacedName{Namespace: podInfo.Namespace(), Name: podInfo.Name()} + pod, ok := m.mtPodState[podNamespacedName.String()] + if !ok { + errBuf := fmt.Sprintf("failed to get pod %v with error %v", podNamespacedName, err) + return types.UnexpectedError, errBuf + } + // check the pod labels for Swift V2, enrich the request with the multitenant flag. TBD on the label + if _, ok := pod.Labels[configuration.LabelSwiftV2]; ok { + req.Multitenant = true + } + return types.Success, "" +} + +// GetSWIFTv2IPConfig(podInfo PodInfo) (*PodIpInfo, error) +// GetMultitenantIPConfig returns the IP config for a multitenant pod from the MTPNC CRD +func (m *MockSWIFTv2Middleware) GetSWIFTv2IPConfig(podInfo cns.PodInfo) (*cns.PodIpInfo, error) { + // Check if the MTPNC CRD exists for the pod, if not, return error + mtpncNamespacedName := k8types.NamespacedName{Namespace: podInfo.Namespace(), Name: podInfo.Name()} + mtpnc, ok := m.mtpncState[mtpncNamespacedName.String()] + if !ok { + return nil, errFailedToGetPod + } + + // Check if the MTPNC CRD is ready. If one of the fields is empty, return error + if mtpnc.Status.PrimaryIP == "" || mtpnc.Status.MacAddress == "" || mtpnc.Status.NCID == "" || mtpnc.Status.GatewayIP == "" { + return nil, errMTPNCNotReady + } + podIPInfo := cns.PodIpInfo{} + podIPInfo.PodIPConfig = cns.IPSubnet{ + IPAddress: mtpnc.Status.PrimaryIP, + } + podIPInfo.MACAddress = mtpnc.Status.MacAddress + podIPInfo.AddressType = cns.Multitenant + podIPInfo.IsDefaultInterface = true + + defaultRoute := cns.Route{ + IPAddress: mtpnc.Status.PrimaryIP, + GatewayIPAddress: mtpnc.Status.GatewayIP, + InterfaceToUse: "eth1", + } + podIPInfo.Routes = []cns.Route{defaultRoute} + + return nil, nil +} diff --git a/cns/restserver/ipam.go b/cns/restserver/ipam.go index 54946bacaa..2d1222ff31 100644 --- a/cns/restserver/ipam.go +++ b/cns/restserver/ipam.go @@ -39,16 +39,6 @@ func (service *HTTPRestService) requestIPConfigHandlerHelper(ipconfigsRequest cn // Request is for multitenant pod if ipconfigsRequest.Multitenant { - // Multitenant middleware not set - if service.MultitenantMiddleware == nil { - return &cns.IPConfigsResponse{ - Response: cns.Response{ - ReturnCode: types.FailedToAllocateIPConfig, - Message: fmt.Sprintf("AllocateIPConfig failed: %v, IP config request is %v", errors.New("request is for multitenant pod but multitenant middleware is nil"), ipconfigsRequest), - }, - PodIPInfo: []cns.PodIpInfo{}, - }, errors.New("request is for multitenant pod but multitenant middleware is nil") - } // If pod is multitenant and we failed to grab its MTPNC IP config, return error immediately podIPInfo, err := service.MultitenantMiddleware.GetSWIFTv2IPConfig(podInfo) if err != nil { From 0860321c7584725f281273753e36fd1b42c706fd Mon Sep 17 00:00:00 2001 From: Quang Nguyen Date: Mon, 11 Sep 2023 18:11:14 +0000 Subject: [PATCH 19/85] update unit tests for ipam --- cns/api.go | 2 +- cns/middlewares/SWIFTv2.go | 8 ++-- cns/middlewares/mock/mockSWIFTv2.go | 10 +++-- cns/restserver/internalapi_test.go | 4 ++ cns/restserver/ipam.go | 10 ++--- cns/restserver/ipam_test.go | 59 +++++++++++++++++++++++++++++ 6 files changed, 79 insertions(+), 14 deletions(-) diff --git a/cns/api.go b/cns/api.go index ce794b48e3..54fe088729 100644 --- a/cns/api.go +++ b/cns/api.go @@ -54,7 +54,7 @@ type HTTPService interface { // Middleware interface for testing later on type SWIFTv2Middleware interface { Validator() IPConfigValidator - GetSWIFTv2IPConfig(podInfo PodInfo) (*PodIpInfo, error) + GetSWIFTv2IPConfig(podInfo PodInfo) (PodIpInfo, error) } type IPConfigValidator func(ipConfigsRequest *IPConfigsRequest) (types.ResponseCode, string) diff --git a/cns/middlewares/SWIFTv2.go b/cns/middlewares/SWIFTv2.go index 7da957f9c3..e94f65bc2d 100644 --- a/cns/middlewares/SWIFTv2.go +++ b/cns/middlewares/SWIFTv2.go @@ -59,18 +59,18 @@ func (m *SWIFTv2Middleware) validateMultitenantIPConfigsRequest(req *cns.IPConfi } // GetMultitenantIPConfig returns the IP config for a multitenant pod from the MTPNC CRD -func (m *SWIFTv2Middleware) GetSWIFTv2IPConfig(podInfo cns.PodInfo) (*cns.PodIpInfo, error) { +func (m *SWIFTv2Middleware) GetSWIFTv2IPConfig(podInfo cns.PodInfo) (cns.PodIpInfo, error) { // Check if the MTPNC CRD exists for the pod, if not, return error mtpnc := v1alpha1.MultitenantPodNetworkConfig{} mtpncNamespacedName := k8types.NamespacedName{Namespace: podInfo.Namespace(), Name: podInfo.Name()} err := m.cli.Get(context.Background(), mtpncNamespacedName, &mtpnc) if err != nil { - return nil, fmt.Errorf("failed to get pod's mtpnc from cache : %w", err) + return cns.PodIpInfo{}, fmt.Errorf("failed to get pod's mtpnc from cache : %w", err) } // Check if the MTPNC CRD is ready. If one of the fields is empty, return error if mtpnc.Status.PrimaryIP == "" || mtpnc.Status.MacAddress == "" || mtpnc.Status.NCID == "" || mtpnc.Status.GatewayIP == "" { - return nil, ErrMTPNCNotReady + return cns.PodIpInfo{}, ErrMTPNCNotReady } podIPInfo := cns.PodIpInfo{} podIPInfo.PodIPConfig = cns.IPSubnet{ @@ -99,5 +99,5 @@ func (m *SWIFTv2Middleware) GetSWIFTv2IPConfig(podInfo cns.PodInfo) (*cns.PodIpI } podIPInfo.Routes = []cns.Route{defaultRoute, podCIDRRoute, serviceCIDRRoute} - return &podIPInfo, nil + return podIPInfo, nil } diff --git a/cns/middlewares/mock/mockSWIFTv2.go b/cns/middlewares/mock/mockSWIFTv2.go index 4d0be95df1..70c3d9e6ec 100644 --- a/cns/middlewares/mock/mockSWIFTv2.go +++ b/cns/middlewares/mock/mockSWIFTv2.go @@ -30,6 +30,8 @@ func NewMockSWIFTv2Middleware() *MockSWIFTv2Middleware { testMTPNC1 := v1alpha1.MultitenantPodNetworkConfig{} testMTPNC1.Status.PrimaryIP = "192.168.0.1" testMTPNC1.Status.MacAddress = "00:00:00:00:00:00" + testMTPNC1.Status.GatewayIP = "10.0.0.1" + testMTPNC1.Status.NCID = "testncid" return &MockSWIFTv2Middleware{ mtPodState: map[string]*v1.Pod{"testpod1namespace/testpod1": &testPod1}, @@ -66,17 +68,17 @@ func (m *MockSWIFTv2Middleware) validateMultitenantIPConfigsRequest(req *cns.IPC // GetSWIFTv2IPConfig(podInfo PodInfo) (*PodIpInfo, error) // GetMultitenantIPConfig returns the IP config for a multitenant pod from the MTPNC CRD -func (m *MockSWIFTv2Middleware) GetSWIFTv2IPConfig(podInfo cns.PodInfo) (*cns.PodIpInfo, error) { +func (m *MockSWIFTv2Middleware) GetSWIFTv2IPConfig(podInfo cns.PodInfo) (cns.PodIpInfo, error) { // Check if the MTPNC CRD exists for the pod, if not, return error mtpncNamespacedName := k8types.NamespacedName{Namespace: podInfo.Namespace(), Name: podInfo.Name()} mtpnc, ok := m.mtpncState[mtpncNamespacedName.String()] if !ok { - return nil, errFailedToGetPod + return cns.PodIpInfo{}, errFailedToGetPod } // Check if the MTPNC CRD is ready. If one of the fields is empty, return error if mtpnc.Status.PrimaryIP == "" || mtpnc.Status.MacAddress == "" || mtpnc.Status.NCID == "" || mtpnc.Status.GatewayIP == "" { - return nil, errMTPNCNotReady + return cns.PodIpInfo{}, errMTPNCNotReady } podIPInfo := cns.PodIpInfo{} podIPInfo.PodIPConfig = cns.IPSubnet{ @@ -93,5 +95,5 @@ func (m *MockSWIFTv2Middleware) GetSWIFTv2IPConfig(podInfo cns.PodInfo) (*cns.Po } podIPInfo.Routes = []cns.Route{defaultRoute} - return nil, nil + return podIPInfo, nil } diff --git a/cns/restserver/internalapi_test.go b/cns/restserver/internalapi_test.go index 854fb3bc81..34ec365d9a 100644 --- a/cns/restserver/internalapi_test.go +++ b/cns/restserver/internalapi_test.go @@ -30,6 +30,10 @@ import ( const ( primaryIp = "10.0.0.5" + SWIFTv2IP = "192.168.0.1" + SWIFTv2MAC = "00:00:00:00:00:00" + SWIFTv2GatewayIP = "10.0.0.1" + SWIFTv2NCID = "testncid" gatewayIp = "10.0.0.1" subnetPrfixLength = 24 dockerContainerType = cns.Docker diff --git a/cns/restserver/ipam.go b/cns/restserver/ipam.go index 2d1222ff31..0eed1c2e6d 100644 --- a/cns/restserver/ipam.go +++ b/cns/restserver/ipam.go @@ -35,10 +35,10 @@ func (service *HTTPRestService) requestIPConfigHandlerHelper(ipconfigsRequest cn }, errors.New("failed to validate ip config request") } - var MTNpodIPInfo *cns.PodIpInfo + var MTNpodIPInfo cns.PodIpInfo // Request is for multitenant pod - if ipconfigsRequest.Multitenant { + if podInfo.IsMultitenant() { // If pod is multitenant and we failed to grab its MTPNC IP config, return error immediately podIPInfo, err := service.MultitenantMiddleware.GetSWIFTv2IPConfig(podInfo) if err != nil { @@ -89,9 +89,9 @@ func (service *HTTPRestService) requestIPConfigHandlerHelper(ipconfigsRequest cn } } - // Adding MTNpodIPInfo to the response, skipping over updateEndpointState since not sure if we have to update endpoint state for MTNpodIPInfo - if MTNpodIPInfo != nil { - podIPInfo = append(podIPInfo, *MTNpodIPInfo) + // Adding MTNpodIPInfo to the response, if it is not empty + if MTNpodIPInfo.PodIPConfig.IPAddress != "" { + podIPInfo = append(podIPInfo, MTNpodIPInfo) } return &cns.IPConfigsResponse{ diff --git a/cns/restserver/ipam_test.go b/cns/restserver/ipam_test.go index b552b15774..7c8d32872e 100644 --- a/cns/restserver/ipam_test.go +++ b/cns/restserver/ipam_test.go @@ -13,6 +13,7 @@ import ( "github.com/Azure/azure-container-networking/cns" "github.com/Azure/azure-container-networking/cns/common" "github.com/Azure/azure-container-networking/cns/fakes" + middlewares "github.com/Azure/azure-container-networking/cns/middlewares/mock" "github.com/Azure/azure-container-networking/cns/types" "github.com/Azure/azure-container-networking/crd/nodenetworkconfig/api/v1alpha" "github.com/Azure/azure-container-networking/store" @@ -65,6 +66,7 @@ func getTestService() *HTTPRestService { var config common.ServiceConfig httpsvc, _ := NewHTTPRestService(&config, &fakes.WireserverClientFake{}, &fakes.WireserverProxyFake{}, &fakes.NMAgentClientFake{}, store.NewMockStore(""), nil, nil) svc = httpsvc + svc.ipConfigsValidators = append(svc.ipConfigsValidators, svc.validateDefaultIPConfigsRequest) httpsvc.IPAMPoolMonitor = &fakes.MonitorFake{} setOrchestratorTypeInternal(cns.KubernetesCRD) @@ -1529,3 +1531,60 @@ func TestIPAMFailToRequestPartialIPsInPool(t *testing.T) { t.Fatalf("Expected fail requesting IPs due to only having one in the ipconfig map, IPs in the pool will not be assigned") } } + +func TestIPAMGetSWIFTv2IP(t *testing.T) { + svc := getTestService() + svc.AttachSWIFTv2Middleware(middlewares.NewMockSWIFTv2Middleware()) + + ncStates := []ncState{ + { + ncID: testNCID, + ips: []string{ + testIP1, + }, + }, + { + ncID: testNCIDv6, + ips: []string{ + testIP1v6, + }, + }, + } + + // Add Available Pod IP to state + for i := range ncStates { + ipconfigs := make(map[string]cns.IPConfigurationStatus, 0) + state := NewPodState(ncStates[i].ips[0], ipIDs[i][0], ncStates[i].ncID, types.Available, 0) + ipconfigs[state.ID] = state + err := UpdatePodIPConfigState(t, svc, ipconfigs, ncStates[i].ncID) + if err != nil { + t.Fatalf("Expected to not fail adding IPs to state: %+v", err) + } + } + + req := cns.IPConfigsRequest{ + PodInterfaceID: testPod1Info.InterfaceID(), + InfraContainerID: testPod1Info.InfraContainerID(), + } + b, _ := testPod1Info.OrchestratorContext() + req.OrchestratorContext = b + req.DesiredIPAddresses = make([]string, 2) + req.DesiredIPAddresses[0] = testIP1 + req.DesiredIPAddresses[1] = testIP1v6 + + resp, err := svc.requestIPConfigHandlerHelper(req) + if err != nil { + t.Fatalf("Expected to not fail requesting IPs: %+v", err) + } + podIPInfo := resp.PodIPInfo + + if len(podIPInfo) != 3 { + t.Fatalf("Expected to get 3 pod IP info (IPv4, IPv6, Multitenant IP), actual %d", len(podIPInfo)) + } + + // Asserting that multitenant IP is returned + assert.Equal(t, SWIFTv2IP, podIPInfo[2].PodIPConfig.IPAddress) + assert.Equal(t, SWIFTv2MAC, podIPInfo[2].MACAddress) + assert.Equal(t, cns.Multitenant, podIPInfo[2].AddressType) + assert.True(t, podIPInfo[2].IsDefaultInterface) +} From 5a89c510f3eb7da583c2bf857d2b1c7bbfa795c7 Mon Sep 17 00:00:00 2001 From: Quang Nguyen Date: Mon, 11 Sep 2023 18:18:46 +0000 Subject: [PATCH 20/85] go get to fix linter --- go.sum | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/go.sum b/go.sum index 500517f1a3..4d38d5888d 100644 --- a/go.sum +++ b/go.sum @@ -315,6 +315,7 @@ github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2 github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= @@ -502,6 +503,7 @@ github.com/joefitzgerald/rainbow-reporter v0.1.0/go.mod h1:481CNgqmVHQZzdIbN52Cu github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= @@ -511,6 +513,7 @@ github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= @@ -596,6 +599,7 @@ github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8m github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= github.com/ncw/swift v1.0.47/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= From e193b82d0e8829f087f17bd5727f8e82ab7cfd3c Mon Sep 17 00:00:00 2001 From: Quang Nguyen Date: Mon, 11 Sep 2023 18:25:22 +0000 Subject: [PATCH 21/85] go mod tidy --- go.mod | 5 ++++- go.sum | 5 ----- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 14d3363e13..d7cfbe11c0 100644 --- a/go.mod +++ b/go.mod @@ -133,7 +133,10 @@ require ( sigs.k8s.io/structured-merge-diff/v4 v4.3.0 // indirect ) -require sigs.k8s.io/yaml v1.3.0 +require ( + gotest.tools/v3 v3.0.3 + sigs.k8s.io/yaml v1.3.0 +) require github.com/emicklei/go-restful/v3 v3.9.0 // indirect diff --git a/go.sum b/go.sum index 4d38d5888d..c98eea69a7 100644 --- a/go.sum +++ b/go.sum @@ -315,7 +315,6 @@ github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2 github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= -github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= @@ -503,7 +502,6 @@ github.com/joefitzgerald/rainbow-reporter v0.1.0/go.mod h1:481CNgqmVHQZzdIbN52Cu github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= -github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= @@ -513,7 +511,6 @@ github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= -github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= @@ -599,7 +596,6 @@ github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8m github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= github.com/ncw/swift v1.0.47/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= @@ -1294,7 +1290,6 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= gotest.tools/v3 v3.0.3 h1:4AuOwCGf4lLR9u3YOe2awrHygurzhO/HeQ6laiA6Sx0= From d1db1ab9c424ef736a2675704ea4afe661a05291 Mon Sep 17 00:00:00 2001 From: Quang Nguyen Date: Mon, 11 Sep 2023 22:18:20 +0000 Subject: [PATCH 22/85] update routes --- cns/configuration/env.go | 26 ++++++++++++++++----- cns/middlewares/SWIFTv2.go | 36 ++++++++++++++++++++++++------ cns/middlewares/SWIFTv2_test.go | 6 +++++ cns/middlewares/mock/mockClient.go | 19 ++++++++++++++++ 4 files changed, 74 insertions(+), 13 deletions(-) diff --git a/cns/configuration/env.go b/cns/configuration/env.go index a6a77f66f5..539ad558b6 100644 --- a/cns/configuration/env.go +++ b/cns/configuration/env.go @@ -13,13 +13,19 @@ const ( EnvNodeIP = "NODE_IP" // LabelSwiftV2 is the Node label for Swift V2 LabelSwiftV2 = "kubernetes.azure.com/podnetwork-multi-tenancy" - EnvPodCIDR = "PodCIDR" - EnvServiceCIDR = "ServiceCIDR" + EnvPodCIDR = "POD_CIDR" + EnvServiceCIDR = "SERVICE_CIDR" ) // ErrNodeNameUnset indicates the the $EnvNodeName variable is unset in the environment. var ErrNodeNameUnset = errors.Errorf("must declare %s environment variable", EnvNodeName) +// ErrNodeIPUnset indicates the the $EnvNodeIP variable is unset in the environment. +var ErrPodCIDRUnset = errors.Errorf("must declare %s environment variable", EnvPodCIDR) + +// ErrNodeIPUnset indicates the the $EnvNodeIP variable is unset in the environment. +var ErrServiceCIDRUnset = errors.Errorf("must declare %s environment variable", EnvPodCIDR) + // NodeName checks the environment variables for the NODENAME and returns it or an error if unset. func NodeName() (string, error) { nodeName := os.Getenv(EnvNodeName) @@ -34,10 +40,18 @@ func NodeIP() string { return os.Getenv(EnvNodeIP) } -func PodCIDR() string { - return os.Getenv(EnvPodCIDR) +func PodCIDR() (string, error) { + podCIDR := os.Getenv(EnvPodCIDR) + if podCIDR == "" { + return "", ErrPodCIDRUnset + } + return podCIDR, nil } -func ServiceCIDR() string { - return os.Getenv(EnvServiceCIDR) +func ServiceCIDR() (string, error) { + serviceCIDR := os.Getenv(EnvServiceCIDR) + if serviceCIDR == "" { + return "", ErrServiceCIDRUnset + } + return serviceCIDR, nil } diff --git a/cns/middlewares/SWIFTv2.go b/cns/middlewares/SWIFTv2.go index e94f65bc2d..dd8595fd71 100644 --- a/cns/middlewares/SWIFTv2.go +++ b/cns/middlewares/SWIFTv2.go @@ -9,8 +9,9 @@ import ( "github.com/Azure/azure-container-networking/cns/configuration" "github.com/Azure/azure-container-networking/cns/types" "github.com/Azure/azure-container-networking/crd/multitenancy/api/v1alpha1" + "github.com/Azure/azure-container-networking/crd/nodenetworkconfig/api/v1alpha" v1 "k8s.io/api/core/v1" - k8types "k8s.io/apimachinery/pkg/types" + k8stypes "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -43,7 +44,7 @@ func (m *SWIFTv2Middleware) validateMultitenantIPConfigsRequest(req *cns.IPConfi errBuf := fmt.Sprintf("unmarshalling pod info from ipconfigs request %v failed with error %v", req, err) return types.UnexpectedError, errBuf } - podNamespacedName := k8types.NamespacedName{Namespace: podInfo.Namespace(), Name: podInfo.Name()} + podNamespacedName := k8stypes.NamespacedName{Namespace: podInfo.Namespace(), Name: podInfo.Name()} pod := v1.Pod{} err = m.cli.Get(context.TODO(), podNamespacedName, &pod) if err != nil { @@ -62,7 +63,7 @@ func (m *SWIFTv2Middleware) validateMultitenantIPConfigsRequest(req *cns.IPConfi func (m *SWIFTv2Middleware) GetSWIFTv2IPConfig(podInfo cns.PodInfo) (cns.PodIpInfo, error) { // Check if the MTPNC CRD exists for the pod, if not, return error mtpnc := v1alpha1.MultitenantPodNetworkConfig{} - mtpncNamespacedName := k8types.NamespacedName{Namespace: podInfo.Namespace(), Name: podInfo.Name()} + mtpncNamespacedName := k8stypes.NamespacedName{Namespace: podInfo.Namespace(), Name: podInfo.Name()} err := m.cli.Get(context.Background(), mtpncNamespacedName, &mtpnc) if err != nil { return cns.PodIpInfo{}, fmt.Errorf("failed to get pod's mtpnc from cache : %w", err) @@ -86,15 +87,36 @@ func (m *SWIFTv2Middleware) GetSWIFTv2IPConfig(podInfo cns.PodInfo) (cns.PodIpIn InterfaceToUse: "eth1", } + nnc := v1alpha.NodeNetworkConfig{} + nodeName, err := configuration.NodeName() + if err != nil { + return cns.PodIpInfo{}, fmt.Errorf("failed to get node name : %w", err) + } + nncNamespacedName := k8stypes.NamespacedName{Namespace: "kube-system", Name: nodeName} + err = m.cli.Get(context.Background(), nncNamespacedName, &nnc) + if err != nil { + return cns.PodIpInfo{}, fmt.Errorf("failed to get node network config : %w", err) + } + + podCIDR, err := configuration.PodCIDR() + if err != nil { + return cns.PodIpInfo{}, fmt.Errorf("failed to get pod CIDR from environment : %w", err) + } + podCIDRRoute := cns.Route{ - IPAddress: configuration.PodCIDR(), - GatewayIPAddress: mtpnc.Status.GatewayIP, + IPAddress: podCIDR, + GatewayIPAddress: nnc.Status.NetworkContainers[0].DefaultGateway, InterfaceToUse: "eth0", } + serviceCIDR, err := configuration.ServiceCIDR() + if err != nil { + return cns.PodIpInfo{}, fmt.Errorf("failed to get service CIDR from environment : %w", err) + } + serviceCIDRRoute := cns.Route{ - IPAddress: configuration.ServiceCIDR(), - GatewayIPAddress: mtpnc.Status.GatewayIP, + IPAddress: serviceCIDR, + GatewayIPAddress: nnc.Status.NetworkContainers[0].DefaultGateway, InterfaceToUse: "eth0", } podIPInfo.Routes = []cns.Route{defaultRoute, podCIDRRoute, serviceCIDRRoute} diff --git a/cns/middlewares/SWIFTv2_test.go b/cns/middlewares/SWIFTv2_test.go index 25515b4bf8..5ba48719a2 100644 --- a/cns/middlewares/SWIFTv2_test.go +++ b/cns/middlewares/SWIFTv2_test.go @@ -1,9 +1,11 @@ package middlewares import ( + "os" "testing" "github.com/Azure/azure-container-networking/cns" + "github.com/Azure/azure-container-networking/cns/configuration" mock "github.com/Azure/azure-container-networking/cns/middlewares/mock" "github.com/Azure/azure-container-networking/cns/types" "gotest.tools/v3/assert" @@ -65,6 +67,10 @@ func TestValidateMultitenantIPConfigsRequestFailure(t *testing.T) { } func TestGetSWIFTv2IPConfigSuccess(t *testing.T) { + os.Setenv(configuration.EnvNodeName, "testnode") + os.Setenv(configuration.EnvPodCIDR, "10.0.1.10/24") + os.Setenv(configuration.EnvServiceCIDR, "10.0.2.10/24") + middleware := NewSWIFTv2Middleware(mock.NewMockClient()) ipInfo, err := middleware.GetSWIFTv2IPConfig(testPod1Info) diff --git a/cns/middlewares/mock/mockClient.go b/cns/middlewares/mock/mockClient.go index c99ef8f828..cf3e68d5c1 100644 --- a/cns/middlewares/mock/mockClient.go +++ b/cns/middlewares/mock/mockClient.go @@ -6,6 +6,7 @@ import ( "github.com/Azure/azure-container-networking/cns/configuration" "github.com/Azure/azure-container-networking/crd/multitenancy/api/v1alpha1" + "github.com/Azure/azure-container-networking/crd/nodenetworkconfig/api/v1alpha" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/runtime" @@ -16,12 +17,14 @@ import ( var ( errPodNotFound = errors.New("pod not found") errMTPNCNotFound = errors.New("mtpnc not found") + errNNCNotFound = errors.New("nnc not found") ) // MockClient implements the client.Client interface for testing. We only care about Get. type MockClient struct { mtPodCache map[string]*v1.Pod mtpncCache map[string]*v1alpha1.MultitenantPodNetworkConfig + nncCache map[string]*v1alpha.NodeNetworkConfig } // NewMockClient returns a new MockClient. @@ -38,12 +41,22 @@ func NewMockClient() *MockClient { testMTPNC3 := v1alpha1.MultitenantPodNetworkConfig{} + testNNC := v1alpha.NodeNetworkConfig{ + Status: v1alpha.NodeNetworkConfigStatus{ + NetworkContainers: []v1alpha.NetworkContainer{ + { + DefaultGateway: "1.1.1.1", + }, + }, + }, + } return &MockClient{ mtPodCache: map[string]*v1.Pod{"testpod1namespace/testpod1": &testPod1}, mtpncCache: map[string]*v1alpha1.MultitenantPodNetworkConfig{ "testpod1namespace/testpod1": &testMTPNC1, "testpod3namespace/testpod3": &testMTPNC3, }, + nncCache: map[string]*v1alpha.NodeNetworkConfig{"kube-system/testnode": &testNNC}, } } @@ -62,6 +75,12 @@ func (c *MockClient) Get(_ context.Context, key client.ObjectKey, obj client.Obj } else { return errMTPNCNotFound } + case *v1alpha.NodeNetworkConfig: + if nnc, ok := c.nncCache[key.String()]; ok { + *o = *nnc + } else { + return errNNCNotFound + } } return nil } From c3e470a48e000dd9a0bae609a11e8aec8b52a404 Mon Sep 17 00:00:00 2001 From: Quang Nguyen Date: Mon, 11 Sep 2023 23:00:17 +0000 Subject: [PATCH 23/85] update routes --- cns/middlewares/SWIFTv2.go | 22 ++++------------------ cns/middlewares/SWIFTv2_test.go | 1 - cns/middlewares/mock/mockClient.go | 21 +-------------------- cns/middlewares/mock/mockSWIFTv2.go | 11 ++++++++++- cns/restserver/ipam.go | 15 +++++++++++---- 5 files changed, 26 insertions(+), 44 deletions(-) diff --git a/cns/middlewares/SWIFTv2.go b/cns/middlewares/SWIFTv2.go index dd8595fd71..6b227649e5 100644 --- a/cns/middlewares/SWIFTv2.go +++ b/cns/middlewares/SWIFTv2.go @@ -9,7 +9,6 @@ import ( "github.com/Azure/azure-container-networking/cns/configuration" "github.com/Azure/azure-container-networking/cns/types" "github.com/Azure/azure-container-networking/crd/multitenancy/api/v1alpha1" - "github.com/Azure/azure-container-networking/crd/nodenetworkconfig/api/v1alpha" v1 "k8s.io/api/core/v1" k8stypes "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" @@ -87,26 +86,14 @@ func (m *SWIFTv2Middleware) GetSWIFTv2IPConfig(podInfo cns.PodInfo) (cns.PodIpIn InterfaceToUse: "eth1", } - nnc := v1alpha.NodeNetworkConfig{} - nodeName, err := configuration.NodeName() - if err != nil { - return cns.PodIpInfo{}, fmt.Errorf("failed to get node name : %w", err) - } - nncNamespacedName := k8stypes.NamespacedName{Namespace: "kube-system", Name: nodeName} - err = m.cli.Get(context.Background(), nncNamespacedName, &nnc) - if err != nil { - return cns.PodIpInfo{}, fmt.Errorf("failed to get node network config : %w", err) - } - podCIDR, err := configuration.PodCIDR() if err != nil { return cns.PodIpInfo{}, fmt.Errorf("failed to get pod CIDR from environment : %w", err) } podCIDRRoute := cns.Route{ - IPAddress: podCIDR, - GatewayIPAddress: nnc.Status.NetworkContainers[0].DefaultGateway, - InterfaceToUse: "eth0", + IPAddress: podCIDR, + InterfaceToUse: "eth0", } serviceCIDR, err := configuration.ServiceCIDR() @@ -115,9 +102,8 @@ func (m *SWIFTv2Middleware) GetSWIFTv2IPConfig(podInfo cns.PodInfo) (cns.PodIpIn } serviceCIDRRoute := cns.Route{ - IPAddress: serviceCIDR, - GatewayIPAddress: nnc.Status.NetworkContainers[0].DefaultGateway, - InterfaceToUse: "eth0", + IPAddress: serviceCIDR, + InterfaceToUse: "eth0", } podIPInfo.Routes = []cns.Route{defaultRoute, podCIDRRoute, serviceCIDRRoute} diff --git a/cns/middlewares/SWIFTv2_test.go b/cns/middlewares/SWIFTv2_test.go index 5ba48719a2..e60772ab94 100644 --- a/cns/middlewares/SWIFTv2_test.go +++ b/cns/middlewares/SWIFTv2_test.go @@ -67,7 +67,6 @@ func TestValidateMultitenantIPConfigsRequestFailure(t *testing.T) { } func TestGetSWIFTv2IPConfigSuccess(t *testing.T) { - os.Setenv(configuration.EnvNodeName, "testnode") os.Setenv(configuration.EnvPodCIDR, "10.0.1.10/24") os.Setenv(configuration.EnvServiceCIDR, "10.0.2.10/24") diff --git a/cns/middlewares/mock/mockClient.go b/cns/middlewares/mock/mockClient.go index cf3e68d5c1..466b34b36a 100644 --- a/cns/middlewares/mock/mockClient.go +++ b/cns/middlewares/mock/mockClient.go @@ -6,7 +6,6 @@ import ( "github.com/Azure/azure-container-networking/cns/configuration" "github.com/Azure/azure-container-networking/crd/multitenancy/api/v1alpha1" - "github.com/Azure/azure-container-networking/crd/nodenetworkconfig/api/v1alpha" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/runtime" @@ -17,14 +16,12 @@ import ( var ( errPodNotFound = errors.New("pod not found") errMTPNCNotFound = errors.New("mtpnc not found") - errNNCNotFound = errors.New("nnc not found") ) -// MockClient implements the client.Client interface for testing. We only care about Get. +// MockClient implements the client.Client interface for testing. We only care about Get, the rest is nil ops. type MockClient struct { mtPodCache map[string]*v1.Pod mtpncCache map[string]*v1alpha1.MultitenantPodNetworkConfig - nncCache map[string]*v1alpha.NodeNetworkConfig } // NewMockClient returns a new MockClient. @@ -41,22 +38,12 @@ func NewMockClient() *MockClient { testMTPNC3 := v1alpha1.MultitenantPodNetworkConfig{} - testNNC := v1alpha.NodeNetworkConfig{ - Status: v1alpha.NodeNetworkConfigStatus{ - NetworkContainers: []v1alpha.NetworkContainer{ - { - DefaultGateway: "1.1.1.1", - }, - }, - }, - } return &MockClient{ mtPodCache: map[string]*v1.Pod{"testpod1namespace/testpod1": &testPod1}, mtpncCache: map[string]*v1alpha1.MultitenantPodNetworkConfig{ "testpod1namespace/testpod1": &testMTPNC1, "testpod3namespace/testpod3": &testMTPNC3, }, - nncCache: map[string]*v1alpha.NodeNetworkConfig{"kube-system/testnode": &testNNC}, } } @@ -75,12 +62,6 @@ func (c *MockClient) Get(_ context.Context, key client.ObjectKey, obj client.Obj } else { return errMTPNCNotFound } - case *v1alpha.NodeNetworkConfig: - if nnc, ok := c.nncCache[key.String()]; ok { - *o = *nnc - } else { - return errNNCNotFound - } } return nil } diff --git a/cns/middlewares/mock/mockSWIFTv2.go b/cns/middlewares/mock/mockSWIFTv2.go index 70c3d9e6ec..904884d02e 100644 --- a/cns/middlewares/mock/mockSWIFTv2.go +++ b/cns/middlewares/mock/mockSWIFTv2.go @@ -93,7 +93,16 @@ func (m *MockSWIFTv2Middleware) GetSWIFTv2IPConfig(podInfo cns.PodInfo) (cns.Pod GatewayIPAddress: mtpnc.Status.GatewayIP, InterfaceToUse: "eth1", } - podIPInfo.Routes = []cns.Route{defaultRoute} + podCIDRRoute := cns.Route{ + IPAddress: "10.0.1.10/24", + InterfaceToUse: "eth0", + } + + serviceCIDRRoute := cns.Route{ + IPAddress: "10.0.2.10/24", + InterfaceToUse: "eth0", + } + podIPInfo.Routes = []cns.Route{defaultRoute, podCIDRRoute, serviceCIDRRoute} return podIPInfo, nil } diff --git a/cns/restserver/ipam.go b/cns/restserver/ipam.go index 0eed1c2e6d..01252e955a 100644 --- a/cns/restserver/ipam.go +++ b/cns/restserver/ipam.go @@ -35,7 +35,7 @@ func (service *HTTPRestService) requestIPConfigHandlerHelper(ipconfigsRequest cn }, errors.New("failed to validate ip config request") } - var MTNpodIPInfo cns.PodIpInfo + var MTpodIPInfo cns.PodIpInfo // Request is for multitenant pod if podInfo.IsMultitenant() { @@ -50,7 +50,7 @@ func (service *HTTPRestService) requestIPConfigHandlerHelper(ipconfigsRequest cn PodIPInfo: []cns.PodIpInfo{}, }, errors.Wrapf(err, "failed to get multitenant IP config %v", ipconfigsRequest) } - MTNpodIPInfo = podIPInfo + MTpodIPInfo = podIPInfo } // record a pod requesting an IP @@ -90,8 +90,15 @@ func (service *HTTPRestService) requestIPConfigHandlerHelper(ipconfigsRequest cn } // Adding MTNpodIPInfo to the response, if it is not empty - if MTNpodIPInfo.PodIPConfig.IPAddress != "" { - podIPInfo = append(podIPInfo, MTNpodIPInfo) + if MTpodIPInfo.PodIPConfig.IPAddress != "" { + defaultGateway := podIPInfo[0].NetworkContainerPrimaryIPConfig.GatewayIPAddress + // Populate the non-multitenant routes with the default gateway + for i := range MTpodIPInfo.Routes { + if MTpodIPInfo.Routes[i].GatewayIPAddress == "" { + MTpodIPInfo.Routes[i].GatewayIPAddress = defaultGateway + } + } + podIPInfo = append(podIPInfo, MTpodIPInfo) } return &cns.IPConfigsResponse{ From dd45bedd0a9242c934b8bbfde01d5b3fcbfeb128 Mon Sep 17 00:00:00 2001 From: Quang Nguyen Date: Fri, 15 Sep 2023 20:43:51 +0000 Subject: [PATCH 24/85] remove stale comments + remove redundant method --- cns/api.go | 2 +- cns/middlewares/SWIFTv2.go | 10 +--------- cns/middlewares/SWIFTv2_test.go | 10 +++------- cns/middlewares/mock/mockSWIFTv2.go | 7 +------ cns/restserver/restserver.go | 2 +- 5 files changed, 7 insertions(+), 24 deletions(-) diff --git a/cns/api.go b/cns/api.go index 54fe088729..65b725d213 100644 --- a/cns/api.go +++ b/cns/api.go @@ -53,7 +53,7 @@ type HTTPService interface { // Middleware interface for testing later on type SWIFTv2Middleware interface { - Validator() IPConfigValidator + ValidateMultitenantIPConfigsRequest(req *IPConfigsRequest) (respCode types.ResponseCode, message string) GetSWIFTv2IPConfig(podInfo PodInfo) (PodIpInfo, error) } diff --git a/cns/middlewares/SWIFTv2.go b/cns/middlewares/SWIFTv2.go index 6b227649e5..4f6df90e1b 100644 --- a/cns/middlewares/SWIFTv2.go +++ b/cns/middlewares/SWIFTv2.go @@ -17,9 +17,6 @@ import ( var ErrMTPNCNotReady = errors.New("mtpnc is not ready") type SWIFTv2Middleware struct { - // TODO: implement - // need cached scoped client for pods - // need client for MTPNC CRD for x-ref pods cli client.Client } @@ -29,14 +26,9 @@ func NewSWIFTv2Middleware(cli client.Client) *SWIFTv2Middleware { } } -// Return the validator function for the middleware -func (m *SWIFTv2Middleware) Validator() cns.IPConfigValidator { - return m.validateMultitenantIPConfigsRequest -} - // validateMultitenantIPConfigsRequest validates if pod is multitenant // nolint -func (m *SWIFTv2Middleware) validateMultitenantIPConfigsRequest(req *cns.IPConfigsRequest) (respCode types.ResponseCode, message string) { +func (m *SWIFTv2Middleware) ValidateMultitenantIPConfigsRequest(req *cns.IPConfigsRequest) (respCode types.ResponseCode, message string) { // Retrieve the pod from the cluster podInfo, err := cns.UnmarshalPodInfo(req.OrchestratorContext) if err != nil { diff --git a/cns/middlewares/SWIFTv2_test.go b/cns/middlewares/SWIFTv2_test.go index e60772ab94..1b92fb90d9 100644 --- a/cns/middlewares/SWIFTv2_test.go +++ b/cns/middlewares/SWIFTv2_test.go @@ -25,8 +25,6 @@ var ( func TestValidateMultitenantIPConfigsRequestSuccess(t *testing.T) { middleware := NewSWIFTv2Middleware(mock.NewMockClient()) - validator := middleware.Validator() - happyReq := &cns.IPConfigsRequest{ PodInterfaceID: testPod1Info.InterfaceID(), InfraContainerID: testPod1Info.InfraContainerID(), @@ -35,7 +33,7 @@ func TestValidateMultitenantIPConfigsRequestSuccess(t *testing.T) { happyReq.OrchestratorContext = b happyReq.Multitenant = false - respCode, err := validator(happyReq) + respCode, err := middleware.ValidateMultitenantIPConfigsRequest(happyReq) assert.Equal(t, err, "") assert.Equal(t, respCode, types.Success) assert.Equal(t, happyReq.Multitenant, true) @@ -44,15 +42,13 @@ func TestValidateMultitenantIPConfigsRequestSuccess(t *testing.T) { func TestValidateMultitenantIPConfigsRequestFailure(t *testing.T) { middleware := NewSWIFTv2Middleware(mock.NewMockClient()) - validator := middleware.Validator() - // Fail to unmarshal pod info test failReq := &cns.IPConfigsRequest{ PodInterfaceID: testPod1Info.InterfaceID(), InfraContainerID: testPod1Info.InfraContainerID(), } failReq.OrchestratorContext = []byte("invalid") - respCode, _ := validator(failReq) + respCode, _ := middleware.ValidateMultitenantIPConfigsRequest(failReq) assert.Equal(t, respCode, types.UnexpectedError) // Pod doesn't exist in cache test @@ -62,7 +58,7 @@ func TestValidateMultitenantIPConfigsRequestFailure(t *testing.T) { } b, _ := testPod2Info.OrchestratorContext() failReq.OrchestratorContext = b - respCode, _ = validator(failReq) + respCode, _ = middleware.ValidateMultitenantIPConfigsRequest(failReq) assert.Equal(t, respCode, types.UnexpectedError) } diff --git a/cns/middlewares/mock/mockSWIFTv2.go b/cns/middlewares/mock/mockSWIFTv2.go index 904884d02e..fd76cd5e4a 100644 --- a/cns/middlewares/mock/mockSWIFTv2.go +++ b/cns/middlewares/mock/mockSWIFTv2.go @@ -39,14 +39,9 @@ func NewMockSWIFTv2Middleware() *MockSWIFTv2Middleware { } } -// Return the validator function for the middleware -func (m *MockSWIFTv2Middleware) Validator() cns.IPConfigValidator { - return m.validateMultitenantIPConfigsRequest -} - // validateMultitenantIPConfigsRequest validates if pod is multitenant // nolint -func (m *MockSWIFTv2Middleware) validateMultitenantIPConfigsRequest(req *cns.IPConfigsRequest) (respCode types.ResponseCode, message string) { +func (m *MockSWIFTv2Middleware) ValidateMultitenantIPConfigsRequest(req *cns.IPConfigsRequest) (respCode types.ResponseCode, message string) { // Retrieve the pod from the cluster podInfo, err := cns.UnmarshalPodInfo(req.OrchestratorContext) if err != nil { diff --git a/cns/restserver/restserver.go b/cns/restserver/restserver.go index 0c8fdcdf2c..7d70f9f309 100644 --- a/cns/restserver/restserver.go +++ b/cns/restserver/restserver.go @@ -356,5 +356,5 @@ func (service *HTTPRestService) MustGenerateCNIConflistOnce() { func (service *HTTPRestService) AttachSWIFTv2Middleware(middleware cns.SWIFTv2Middleware) { service.MultitenantMiddleware = middleware // add multitenant ipconfig validator function - service.ipConfigsValidators = append(service.ipConfigsValidators, middleware.Validator()) + service.ipConfigsValidators = append(service.ipConfigsValidators, middleware.ValidateMultitenantIPConfigsRequest) } From bf483aaaf361135e23dc4f0f0de6aa9bb015182b Mon Sep 17 00:00:00 2001 From: Quang Nguyen Date: Mon, 18 Sep 2023 21:24:08 +0000 Subject: [PATCH 25/85] add contexts + change address type --- cns/NetworkContainerContract.go | 4 ++-- cns/api.go | 4 ++-- cns/middlewares/SWIFTv2.go | 6 +++--- cns/middlewares/SWIFTv2_test.go | 9 +++++---- cns/middlewares/mock/mockSWIFTv2.go | 5 +++-- cns/restserver/ipam.go | 9 +++++---- cns/restserver/ipam_test.go | 5 +++-- 7 files changed, 23 insertions(+), 19 deletions(-) diff --git a/cns/NetworkContainerContract.go b/cns/NetworkContainerContract.go index f404329845..4994b42d16 100644 --- a/cns/NetworkContainerContract.go +++ b/cns/NetworkContainerContract.go @@ -79,8 +79,8 @@ const ( // Address Types const ( - Default = "Default" - Multitenant = "Multitenant" + Default = "Default" + Secondary = "Secondary" ) // CreateNetworkContainerRequest specifies request to create a network container or network isolation boundary. diff --git a/cns/api.go b/cns/api.go index 65b725d213..7a4f8c75e1 100644 --- a/cns/api.go +++ b/cns/api.go @@ -53,8 +53,8 @@ type HTTPService interface { // Middleware interface for testing later on type SWIFTv2Middleware interface { - ValidateMultitenantIPConfigsRequest(req *IPConfigsRequest) (respCode types.ResponseCode, message string) - GetSWIFTv2IPConfig(podInfo PodInfo) (PodIpInfo, error) + ValidateMultitenantIPConfigsRequest(*IPConfigsRequest) (types.ResponseCode, string) + GetSWIFTv2IPConfig(context.Context, PodInfo) (PodIpInfo, error) } type IPConfigValidator func(ipConfigsRequest *IPConfigsRequest) (types.ResponseCode, string) diff --git a/cns/middlewares/SWIFTv2.go b/cns/middlewares/SWIFTv2.go index 4f6df90e1b..7ecef6e3c2 100644 --- a/cns/middlewares/SWIFTv2.go +++ b/cns/middlewares/SWIFTv2.go @@ -51,11 +51,11 @@ func (m *SWIFTv2Middleware) ValidateMultitenantIPConfigsRequest(req *cns.IPConfi } // GetMultitenantIPConfig returns the IP config for a multitenant pod from the MTPNC CRD -func (m *SWIFTv2Middleware) GetSWIFTv2IPConfig(podInfo cns.PodInfo) (cns.PodIpInfo, error) { +func (m *SWIFTv2Middleware) GetSWIFTv2IPConfig(ctx context.Context, podInfo cns.PodInfo) (cns.PodIpInfo, error) { // Check if the MTPNC CRD exists for the pod, if not, return error mtpnc := v1alpha1.MultitenantPodNetworkConfig{} mtpncNamespacedName := k8stypes.NamespacedName{Namespace: podInfo.Namespace(), Name: podInfo.Name()} - err := m.cli.Get(context.Background(), mtpncNamespacedName, &mtpnc) + err := m.cli.Get(ctx, mtpncNamespacedName, &mtpnc) if err != nil { return cns.PodIpInfo{}, fmt.Errorf("failed to get pod's mtpnc from cache : %w", err) } @@ -69,7 +69,7 @@ func (m *SWIFTv2Middleware) GetSWIFTv2IPConfig(podInfo cns.PodInfo) (cns.PodIpIn IPAddress: mtpnc.Status.PrimaryIP, } podIPInfo.MACAddress = mtpnc.Status.MacAddress - podIPInfo.AddressType = cns.Multitenant + podIPInfo.AddressType = cns.Secondary podIPInfo.IsDefaultInterface = true defaultRoute := cns.Route{ diff --git a/cns/middlewares/SWIFTv2_test.go b/cns/middlewares/SWIFTv2_test.go index 1b92fb90d9..77aa02ff4c 100644 --- a/cns/middlewares/SWIFTv2_test.go +++ b/cns/middlewares/SWIFTv2_test.go @@ -1,6 +1,7 @@ package middlewares import ( + "context" "os" "testing" @@ -68,9 +69,9 @@ func TestGetSWIFTv2IPConfigSuccess(t *testing.T) { middleware := NewSWIFTv2Middleware(mock.NewMockClient()) - ipInfo, err := middleware.GetSWIFTv2IPConfig(testPod1Info) + ipInfo, err := middleware.GetSWIFTv2IPConfig(context.TODO(), testPod1Info) assert.Equal(t, err, nil) - assert.Equal(t, ipInfo.AddressType, cns.Multitenant) + assert.Equal(t, ipInfo.AddressType, cns.Secondary) assert.Equal(t, ipInfo.IsDefaultInterface, true) } @@ -78,10 +79,10 @@ func TestGetSWIFTv2IPConfigFailure(t *testing.T) { middleware := NewSWIFTv2Middleware(mock.NewMockClient()) // Pod's MTPNC doesn't exist in cache test - _, err := middleware.GetSWIFTv2IPConfig(testPod2Info) + _, err := middleware.GetSWIFTv2IPConfig(context.TODO(), testPod2Info) assert.Error(t, err, "failed to get pod's mtpnc from cache : mtpnc not found") // Pod's MTPNC is not ready test - _, err = middleware.GetSWIFTv2IPConfig(testPod3Info) + _, err = middleware.GetSWIFTv2IPConfig(context.TODO(), testPod3Info) assert.Error(t, err, ErrMTPNCNotReady.Error()) } diff --git a/cns/middlewares/mock/mockSWIFTv2.go b/cns/middlewares/mock/mockSWIFTv2.go index fd76cd5e4a..6793c5b142 100644 --- a/cns/middlewares/mock/mockSWIFTv2.go +++ b/cns/middlewares/mock/mockSWIFTv2.go @@ -1,6 +1,7 @@ package middlewares import ( + "context" "errors" "fmt" @@ -63,7 +64,7 @@ func (m *MockSWIFTv2Middleware) ValidateMultitenantIPConfigsRequest(req *cns.IPC // GetSWIFTv2IPConfig(podInfo PodInfo) (*PodIpInfo, error) // GetMultitenantIPConfig returns the IP config for a multitenant pod from the MTPNC CRD -func (m *MockSWIFTv2Middleware) GetSWIFTv2IPConfig(podInfo cns.PodInfo) (cns.PodIpInfo, error) { +func (m *MockSWIFTv2Middleware) GetSWIFTv2IPConfig(_ context.Context, podInfo cns.PodInfo) (cns.PodIpInfo, error) { // Check if the MTPNC CRD exists for the pod, if not, return error mtpncNamespacedName := k8types.NamespacedName{Namespace: podInfo.Namespace(), Name: podInfo.Name()} mtpnc, ok := m.mtpncState[mtpncNamespacedName.String()] @@ -80,7 +81,7 @@ func (m *MockSWIFTv2Middleware) GetSWIFTv2IPConfig(podInfo cns.PodInfo) (cns.Pod IPAddress: mtpnc.Status.PrimaryIP, } podIPInfo.MACAddress = mtpnc.Status.MacAddress - podIPInfo.AddressType = cns.Multitenant + podIPInfo.AddressType = cns.Secondary podIPInfo.IsDefaultInterface = true defaultRoute := cns.Route{ diff --git a/cns/restserver/ipam.go b/cns/restserver/ipam.go index 01252e955a..d0ecc3245b 100644 --- a/cns/restserver/ipam.go +++ b/cns/restserver/ipam.go @@ -4,6 +4,7 @@ package restserver import ( + "context" "fmt" "net" "net/http" @@ -24,7 +25,7 @@ var ( ) // requestIPConfigHandlerHelper validates the request, assigns IPs, and returns a response -func (service *HTTPRestService) requestIPConfigHandlerHelper(ipconfigsRequest cns.IPConfigsRequest) (*cns.IPConfigsResponse, error) { +func (service *HTTPRestService) requestIPConfigHandlerHelper(ctx context.Context, ipconfigsRequest cns.IPConfigsRequest) (*cns.IPConfigsResponse, error) { podInfo, returnCode, returnMessage := service.validateIPConfigsRequest(ipconfigsRequest) if returnCode != types.Success { return &cns.IPConfigsResponse{ @@ -40,7 +41,7 @@ func (service *HTTPRestService) requestIPConfigHandlerHelper(ipconfigsRequest cn // Request is for multitenant pod if podInfo.IsMultitenant() { // If pod is multitenant and we failed to grab its MTPNC IP config, return error immediately - podIPInfo, err := service.MultitenantMiddleware.GetSWIFTv2IPConfig(podInfo) + podIPInfo, err := service.MultitenantMiddleware.GetSWIFTv2IPConfig(ctx, podInfo) if err != nil { return &cns.IPConfigsResponse{ Response: cns.Response{ @@ -147,7 +148,7 @@ func (service *HTTPRestService) requestIPConfigHandler(w http.ResponseWriter, r } } - ipConfigsResp, errResp := service.requestIPConfigHandlerHelper(ipconfigsRequest) //nolint:contextcheck // appease linter + ipConfigsResp, errResp := service.requestIPConfigHandlerHelper(context.Background(), ipconfigsRequest) //nolint:contextcheck // appease linter if errResp != nil { // As this API is expected to return IPConfigResponse, generate it from the IPConfigsResponse returned above reserveResp := &cns.IPConfigResponse{ @@ -194,7 +195,7 @@ func (service *HTTPRestService) requestIPConfigsHandler(w http.ResponseWriter, r return } - ipConfigsResp, err := service.requestIPConfigHandlerHelper(ipconfigsRequest) // nolint:contextcheck // appease linter + ipConfigsResp, err := service.requestIPConfigHandlerHelper(context.Background(), ipconfigsRequest) // nolint:contextcheck // appease linter if err != nil { w.Header().Set(cnsReturnCode, ipConfigsResp.Response.ReturnCode.String()) err = service.Listener.Encode(w, &ipConfigsResp) diff --git a/cns/restserver/ipam_test.go b/cns/restserver/ipam_test.go index 7c8d32872e..d4890fa27a 100644 --- a/cns/restserver/ipam_test.go +++ b/cns/restserver/ipam_test.go @@ -4,6 +4,7 @@ package restserver import ( + "context" "fmt" "net" "net/netip" @@ -1572,7 +1573,7 @@ func TestIPAMGetSWIFTv2IP(t *testing.T) { req.DesiredIPAddresses[0] = testIP1 req.DesiredIPAddresses[1] = testIP1v6 - resp, err := svc.requestIPConfigHandlerHelper(req) + resp, err := svc.requestIPConfigHandlerHelper(context.TODO(), req) if err != nil { t.Fatalf("Expected to not fail requesting IPs: %+v", err) } @@ -1585,6 +1586,6 @@ func TestIPAMGetSWIFTv2IP(t *testing.T) { // Asserting that multitenant IP is returned assert.Equal(t, SWIFTv2IP, podIPInfo[2].PodIPConfig.IPAddress) assert.Equal(t, SWIFTv2MAC, podIPInfo[2].MACAddress) - assert.Equal(t, cns.Multitenant, podIPInfo[2].AddressType) + assert.Equal(t, cns.Secondary, podIPInfo[2].AddressType) assert.True(t, podIPInfo[2].IsDefaultInterface) } From 843a64e9214cef2cadea4a54fe8b60e8fa06e124 Mon Sep 17 00:00:00 2001 From: Quang Nguyen Date: Fri, 22 Sep 2023 16:58:17 +0000 Subject: [PATCH 26/85] addressed review --- cns/configuration/env.go | 8 ++++---- cns/middlewares/{SWIFTv2.go => swiftV2.go} | 14 +++----------- .../{SWIFTv2_test.go => swiftV2_test.go} | 8 ++++---- cns/service/main.go | 4 ++-- 4 files changed, 13 insertions(+), 21 deletions(-) rename cns/middlewares/{SWIFTv2.go => swiftV2.go} (92%) rename cns/middlewares/{SWIFTv2_test.go => swiftV2_test.go} (92%) diff --git a/cns/configuration/env.go b/cns/configuration/env.go index 539ad558b6..7af2705952 100644 --- a/cns/configuration/env.go +++ b/cns/configuration/env.go @@ -12,7 +12,7 @@ const ( // EnvNodeIP is the IP of the node running this CNS binary EnvNodeIP = "NODE_IP" // LabelSwiftV2 is the Node label for Swift V2 - LabelSwiftV2 = "kubernetes.azure.com/podnetwork-multi-tenancy" + LabelSwiftV2 = "kubernetes.azure.com/podnetwork-multi-tenancy-enabled" EnvPodCIDR = "POD_CIDR" EnvServiceCIDR = "SERVICE_CIDR" ) @@ -20,11 +20,11 @@ const ( // ErrNodeNameUnset indicates the the $EnvNodeName variable is unset in the environment. var ErrNodeNameUnset = errors.Errorf("must declare %s environment variable", EnvNodeName) -// ErrNodeIPUnset indicates the the $EnvNodeIP variable is unset in the environment. +// ErrPodCIDRUnset indicates the the $EnvPodCIDR variable is unset in the environment. var ErrPodCIDRUnset = errors.Errorf("must declare %s environment variable", EnvPodCIDR) -// ErrNodeIPUnset indicates the the $EnvNodeIP variable is unset in the environment. -var ErrServiceCIDRUnset = errors.Errorf("must declare %s environment variable", EnvPodCIDR) +// ErrServiceCIDRUnset indicates the the $EnvServiceCIDR variable is unset in the environment. +var ErrServiceCIDRUnset = errors.Errorf("must declare %s environment variable", EnvServiceCIDR) // NodeName checks the environment variables for the NODENAME and returns it or an error if unset. func NodeName() (string, error) { diff --git a/cns/middlewares/SWIFTv2.go b/cns/middlewares/swiftV2.go similarity index 92% rename from cns/middlewares/SWIFTv2.go rename to cns/middlewares/swiftV2.go index 7ecef6e3c2..ceef081894 100644 --- a/cns/middlewares/SWIFTv2.go +++ b/cns/middlewares/swiftV2.go @@ -17,13 +17,7 @@ import ( var ErrMTPNCNotReady = errors.New("mtpnc is not ready") type SWIFTv2Middleware struct { - cli client.Client -} - -func NewSWIFTv2Middleware(cli client.Client) *SWIFTv2Middleware { - return &SWIFTv2Middleware{ - cli: cli, - } + Cli client.Client } // validateMultitenantIPConfigsRequest validates if pod is multitenant @@ -37,8 +31,7 @@ func (m *SWIFTv2Middleware) ValidateMultitenantIPConfigsRequest(req *cns.IPConfi } podNamespacedName := k8stypes.NamespacedName{Namespace: podInfo.Namespace(), Name: podInfo.Name()} pod := v1.Pod{} - err = m.cli.Get(context.TODO(), podNamespacedName, &pod) - if err != nil { + if err := m.Cli.Get(context.TODO(), podNamespacedName, &pod); err != nil { errBuf := fmt.Sprintf("failed to get pod %v with error %v", podNamespacedName, err) return types.UnexpectedError, errBuf } @@ -55,8 +48,7 @@ func (m *SWIFTv2Middleware) GetSWIFTv2IPConfig(ctx context.Context, podInfo cns. // Check if the MTPNC CRD exists for the pod, if not, return error mtpnc := v1alpha1.MultitenantPodNetworkConfig{} mtpncNamespacedName := k8stypes.NamespacedName{Namespace: podInfo.Namespace(), Name: podInfo.Name()} - err := m.cli.Get(ctx, mtpncNamespacedName, &mtpnc) - if err != nil { + if err := m.Cli.Get(ctx, mtpncNamespacedName, &mtpnc); err != nil { return cns.PodIpInfo{}, fmt.Errorf("failed to get pod's mtpnc from cache : %w", err) } diff --git a/cns/middlewares/SWIFTv2_test.go b/cns/middlewares/swiftV2_test.go similarity index 92% rename from cns/middlewares/SWIFTv2_test.go rename to cns/middlewares/swiftV2_test.go index 77aa02ff4c..0c222df584 100644 --- a/cns/middlewares/SWIFTv2_test.go +++ b/cns/middlewares/swiftV2_test.go @@ -24,7 +24,7 @@ var ( ) func TestValidateMultitenantIPConfigsRequestSuccess(t *testing.T) { - middleware := NewSWIFTv2Middleware(mock.NewMockClient()) + middleware := SWIFTv2Middleware{Cli: mock.NewMockClient()} happyReq := &cns.IPConfigsRequest{ PodInterfaceID: testPod1Info.InterfaceID(), @@ -41,7 +41,7 @@ func TestValidateMultitenantIPConfigsRequestSuccess(t *testing.T) { } func TestValidateMultitenantIPConfigsRequestFailure(t *testing.T) { - middleware := NewSWIFTv2Middleware(mock.NewMockClient()) + middleware := SWIFTv2Middleware{Cli: mock.NewMockClient()} // Fail to unmarshal pod info test failReq := &cns.IPConfigsRequest{ @@ -67,7 +67,7 @@ func TestGetSWIFTv2IPConfigSuccess(t *testing.T) { os.Setenv(configuration.EnvPodCIDR, "10.0.1.10/24") os.Setenv(configuration.EnvServiceCIDR, "10.0.2.10/24") - middleware := NewSWIFTv2Middleware(mock.NewMockClient()) + middleware := SWIFTv2Middleware{Cli: mock.NewMockClient()} ipInfo, err := middleware.GetSWIFTv2IPConfig(context.TODO(), testPod1Info) assert.Equal(t, err, nil) @@ -76,7 +76,7 @@ func TestGetSWIFTv2IPConfigSuccess(t *testing.T) { } func TestGetSWIFTv2IPConfigFailure(t *testing.T) { - middleware := NewSWIFTv2Middleware(mock.NewMockClient()) + middleware := SWIFTv2Middleware{Cli: mock.NewMockClient()} // Pod's MTPNC doesn't exist in cache test _, err := middleware.GetSWIFTv2IPConfig(context.TODO(), testPod2Info) diff --git a/cns/service/main.go b/cns/service/main.go index b3e8268dce..b903c0278d 100644 --- a/cns/service/main.go +++ b/cns/service/main.go @@ -1324,8 +1324,8 @@ func InitializeCRDState(ctx context.Context, httpRestService cns.HTTPService, cn return errors.Wrapf(err, "failed to setup mtpnc reconciler with manager") } // if SWIFT v2 is enabled on CNS, attach multitenant middleware to rest service - swiftV2Middleware := middlewares.NewSWIFTv2Middleware(manager.GetClient()) - httpRestService.AttachSWIFTv2Middleware(swiftV2Middleware) + swiftV2Middleware := middlewares.SWIFTv2Middleware{Cli: manager.GetClient()} + httpRestService.AttachSWIFTv2Middleware(&swiftV2Middleware) } // adding some routes to the root service mux From 04ee54ca295dbbcd51f2fb8c392cf6b0324e738b Mon Sep 17 00:00:00 2001 From: Quang Nguyen Date: Fri, 22 Sep 2023 20:14:17 +0000 Subject: [PATCH 27/85] embedded client to mock + enum for address type --- cns/NetworkContainerContract.go | 8 ++-- cns/middlewares/mock/mockClient.go | 64 +---------------------------- cns/middlewares/mock/mockSWIFTv2.go | 2 +- cns/middlewares/swiftV2.go | 2 +- cns/middlewares/swiftV2_test.go | 2 +- 5 files changed, 9 insertions(+), 69 deletions(-) diff --git a/cns/NetworkContainerContract.go b/cns/NetworkContainerContract.go index 89c215e26d..bd5a8a34f2 100644 --- a/cns/NetworkContainerContract.go +++ b/cns/NetworkContainerContract.go @@ -78,10 +78,12 @@ const ( MultiTenantCRD = "MultiTenantCRD" ) +type AddressType string + // Address Types const ( - Default = "Default" - Secondary = "Secondary" + AddressTypeDefault AddressType = "Default" + AddressTypeSecondary AddressType = "Secondary" ) // CreateNetworkContainerRequest specifies request to create a network container or network isolation boundary. @@ -420,7 +422,7 @@ type PodIpInfo struct { PodIPConfig IPSubnet NetworkContainerPrimaryIPConfig IPConfiguration HostPrimaryIPInfo HostIPInfo - AddressType string // multitenant or default, maybe make this an enum later + AddressType AddressType // multitenant or default, maybe make this an enum later MACAddress string IsDefaultInterface bool Routes []Route diff --git a/cns/middlewares/mock/mockClient.go b/cns/middlewares/mock/mockClient.go index 466b34b36a..5dac13bc31 100644 --- a/cns/middlewares/mock/mockClient.go +++ b/cns/middlewares/mock/mockClient.go @@ -7,9 +7,6 @@ import ( "github.com/Azure/azure-container-networking/cns/configuration" "github.com/Azure/azure-container-networking/crd/multitenancy/api/v1alpha1" v1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/meta" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -20,6 +17,7 @@ var ( // MockClient implements the client.Client interface for testing. We only care about Get, the rest is nil ops. type MockClient struct { + client.Client mtPodCache map[string]*v1.Pod mtpncCache map[string]*v1alpha1.MultitenantPodNetworkConfig } @@ -65,63 +63,3 @@ func (c *MockClient) Get(_ context.Context, key client.ObjectKey, obj client.Obj } return nil } - -// List implements client.Client.List. -func (c *MockClient) List(_ context.Context, _ client.ObjectList, _ ...client.ListOption) error { - return nil -} - -// Create implements client.Client.Create. -func (c *MockClient) Create(_ context.Context, _ client.Object, _ ...client.CreateOption) error { - return nil -} - -// Delete implements client.Client.Delete. -func (c *MockClient) Delete(_ context.Context, _ client.Object, _ ...client.DeleteOption) error { - return nil -} - -// Update implements client.Client.Update. -func (c *MockClient) Update(_ context.Context, _ client.Object, _ ...client.UpdateOption) error { - return nil -} - -// Patch implements client.Client.Patch. -func (c *MockClient) Patch(_ context.Context, _ client.Object, _ client.Patch, _ ...client.PatchOption) error { - return nil -} - -// DeleteAllOf implements client.Client.DeleteAllOf. -func (c *MockClient) DeleteAllOf(_ context.Context, _ client.Object, _ ...client.DeleteAllOfOption) error { - return nil -} - -// Status implements client.StatusClient. -func (c *MockClient) Status() client.StatusWriter { - return nil -} - -// RESTMapper implements client.Client. -func (c *MockClient) RESTMapper() meta.RESTMapper { - return nil -} - -// Scheme implements client.Client. -func (c *MockClient) Scheme() *runtime.Scheme { - return nil -} - -// GroupVersionKindFor implements client.Client. -func (c *MockClient) GroupVersionKindFor(_ runtime.Object) (schema.GroupVersionKind, error) { - return schema.GroupVersionKind{}, nil -} - -// IsObjectNamespaced implements client.Client. -func (c *MockClient) IsObjectNamespaced(_ runtime.Object) (bool, error) { - return false, nil -} - -// SubResource implements client.Client. -func (c *MockClient) SubResource(_ string) client.SubResourceClient { - return nil -} diff --git a/cns/middlewares/mock/mockSWIFTv2.go b/cns/middlewares/mock/mockSWIFTv2.go index 6793c5b142..c9ad9761be 100644 --- a/cns/middlewares/mock/mockSWIFTv2.go +++ b/cns/middlewares/mock/mockSWIFTv2.go @@ -81,7 +81,7 @@ func (m *MockSWIFTv2Middleware) GetSWIFTv2IPConfig(_ context.Context, podInfo cn IPAddress: mtpnc.Status.PrimaryIP, } podIPInfo.MACAddress = mtpnc.Status.MacAddress - podIPInfo.AddressType = cns.Secondary + podIPInfo.AddressType = cns.AddressTypeSecondary podIPInfo.IsDefaultInterface = true defaultRoute := cns.Route{ diff --git a/cns/middlewares/swiftV2.go b/cns/middlewares/swiftV2.go index ceef081894..29c07cafcd 100644 --- a/cns/middlewares/swiftV2.go +++ b/cns/middlewares/swiftV2.go @@ -61,7 +61,7 @@ func (m *SWIFTv2Middleware) GetSWIFTv2IPConfig(ctx context.Context, podInfo cns. IPAddress: mtpnc.Status.PrimaryIP, } podIPInfo.MACAddress = mtpnc.Status.MacAddress - podIPInfo.AddressType = cns.Secondary + podIPInfo.AddressType = cns.AddressTypeSecondary podIPInfo.IsDefaultInterface = true defaultRoute := cns.Route{ diff --git a/cns/middlewares/swiftV2_test.go b/cns/middlewares/swiftV2_test.go index 0c222df584..deacd1b951 100644 --- a/cns/middlewares/swiftV2_test.go +++ b/cns/middlewares/swiftV2_test.go @@ -71,7 +71,7 @@ func TestGetSWIFTv2IPConfigSuccess(t *testing.T) { ipInfo, err := middleware.GetSWIFTv2IPConfig(context.TODO(), testPod1Info) assert.Equal(t, err, nil) - assert.Equal(t, ipInfo.AddressType, cns.Secondary) + assert.Equal(t, ipInfo.AddressType, cns.AddressTypeSecondary) assert.Equal(t, ipInfo.IsDefaultInterface, true) } From 08b4a7866d5b3738d95a73940fe116d761131d21 Mon Sep 17 00:00:00 2001 From: Quang Nguyen Date: Fri, 22 Sep 2023 20:15:47 +0000 Subject: [PATCH 28/85] fix error --- cns/restserver/ipam_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cns/restserver/ipam_test.go b/cns/restserver/ipam_test.go index d4890fa27a..87a8ff03c1 100644 --- a/cns/restserver/ipam_test.go +++ b/cns/restserver/ipam_test.go @@ -1586,6 +1586,6 @@ func TestIPAMGetSWIFTv2IP(t *testing.T) { // Asserting that multitenant IP is returned assert.Equal(t, SWIFTv2IP, podIPInfo[2].PodIPConfig.IPAddress) assert.Equal(t, SWIFTv2MAC, podIPInfo[2].MACAddress) - assert.Equal(t, cns.Secondary, podIPInfo[2].AddressType) + assert.Equal(t, cns.AddressTypeSecondary, podIPInfo[2].AddressType) assert.True(t, podIPInfo[2].IsDefaultInterface) } From 6bb2b7d125597df00fb7c5ffc6b41b6fb33323cd Mon Sep 17 00:00:00 2001 From: Quang Nguyen Date: Mon, 25 Sep 2023 20:00:43 +0000 Subject: [PATCH 29/85] change addressType to NICType --- cns/NetworkContainerContract.go | 10 +++++----- cns/middlewares/mock/mockSWIFTv2.go | 2 +- cns/middlewares/swiftV2.go | 2 +- cns/middlewares/swiftV2_test.go | 2 +- cns/restserver/ipam_test.go | 2 +- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/cns/NetworkContainerContract.go b/cns/NetworkContainerContract.go index bd5a8a34f2..7b248235bb 100644 --- a/cns/NetworkContainerContract.go +++ b/cns/NetworkContainerContract.go @@ -78,12 +78,12 @@ const ( MultiTenantCRD = "MultiTenantCRD" ) -type AddressType string +type NICType string -// Address Types +// NIC Types const ( - AddressTypeDefault AddressType = "Default" - AddressTypeSecondary AddressType = "Secondary" + NICTypeDefault NICType = "Default" + NICTypeSecondary NICType = "Secondary" ) // CreateNetworkContainerRequest specifies request to create a network container or network isolation boundary. @@ -422,7 +422,7 @@ type PodIpInfo struct { PodIPConfig IPSubnet NetworkContainerPrimaryIPConfig IPConfiguration HostPrimaryIPInfo HostIPInfo - AddressType AddressType // multitenant or default, maybe make this an enum later + NICType NICType // default, secondary MACAddress string IsDefaultInterface bool Routes []Route diff --git a/cns/middlewares/mock/mockSWIFTv2.go b/cns/middlewares/mock/mockSWIFTv2.go index c9ad9761be..848d925262 100644 --- a/cns/middlewares/mock/mockSWIFTv2.go +++ b/cns/middlewares/mock/mockSWIFTv2.go @@ -81,7 +81,7 @@ func (m *MockSWIFTv2Middleware) GetSWIFTv2IPConfig(_ context.Context, podInfo cn IPAddress: mtpnc.Status.PrimaryIP, } podIPInfo.MACAddress = mtpnc.Status.MacAddress - podIPInfo.AddressType = cns.AddressTypeSecondary + podIPInfo.NICType = cns.NICTypeSecondary podIPInfo.IsDefaultInterface = true defaultRoute := cns.Route{ diff --git a/cns/middlewares/swiftV2.go b/cns/middlewares/swiftV2.go index 29c07cafcd..f522f1550f 100644 --- a/cns/middlewares/swiftV2.go +++ b/cns/middlewares/swiftV2.go @@ -61,7 +61,7 @@ func (m *SWIFTv2Middleware) GetSWIFTv2IPConfig(ctx context.Context, podInfo cns. IPAddress: mtpnc.Status.PrimaryIP, } podIPInfo.MACAddress = mtpnc.Status.MacAddress - podIPInfo.AddressType = cns.AddressTypeSecondary + podIPInfo.NICType = cns.NICTypeSecondary podIPInfo.IsDefaultInterface = true defaultRoute := cns.Route{ diff --git a/cns/middlewares/swiftV2_test.go b/cns/middlewares/swiftV2_test.go index deacd1b951..981bca713b 100644 --- a/cns/middlewares/swiftV2_test.go +++ b/cns/middlewares/swiftV2_test.go @@ -71,7 +71,7 @@ func TestGetSWIFTv2IPConfigSuccess(t *testing.T) { ipInfo, err := middleware.GetSWIFTv2IPConfig(context.TODO(), testPod1Info) assert.Equal(t, err, nil) - assert.Equal(t, ipInfo.AddressType, cns.AddressTypeSecondary) + assert.Equal(t, ipInfo.NICType, cns.NICTypeSecondary) assert.Equal(t, ipInfo.IsDefaultInterface, true) } diff --git a/cns/restserver/ipam_test.go b/cns/restserver/ipam_test.go index 87a8ff03c1..662acd1303 100644 --- a/cns/restserver/ipam_test.go +++ b/cns/restserver/ipam_test.go @@ -1586,6 +1586,6 @@ func TestIPAMGetSWIFTv2IP(t *testing.T) { // Asserting that multitenant IP is returned assert.Equal(t, SWIFTv2IP, podIPInfo[2].PodIPConfig.IPAddress) assert.Equal(t, SWIFTv2MAC, podIPInfo[2].MACAddress) - assert.Equal(t, cns.AddressTypeSecondary, podIPInfo[2].AddressType) + assert.Equal(t, cns.NICTypeSecondary, podIPInfo[2].NICType) assert.True(t, podIPInfo[2].IsDefaultInterface) } From 4ee59375380610c3754a71a2781d86b87fc85c52 Mon Sep 17 00:00:00 2001 From: Quang Nguyen Date: Mon, 25 Sep 2023 20:05:55 +0000 Subject: [PATCH 30/85] change isDefaultRoute to SkipDefaultRoutes --- cns/NetworkContainerContract.go | 2 +- cns/middlewares/mock/mockSWIFTv2.go | 2 +- cns/middlewares/swiftV2.go | 2 +- cns/middlewares/swiftV2_test.go | 2 +- cns/restserver/ipam_test.go | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/cns/NetworkContainerContract.go b/cns/NetworkContainerContract.go index 7b248235bb..c66d07ac0e 100644 --- a/cns/NetworkContainerContract.go +++ b/cns/NetworkContainerContract.go @@ -424,7 +424,7 @@ type PodIpInfo struct { HostPrimaryIPInfo HostIPInfo NICType NICType // default, secondary MACAddress string - IsDefaultInterface bool + SkipDefaultRoutes bool // if set then cni should honor routes in PodIPInfo.Routes[] Routes []Route } diff --git a/cns/middlewares/mock/mockSWIFTv2.go b/cns/middlewares/mock/mockSWIFTv2.go index 848d925262..df5bc646d8 100644 --- a/cns/middlewares/mock/mockSWIFTv2.go +++ b/cns/middlewares/mock/mockSWIFTv2.go @@ -82,7 +82,7 @@ func (m *MockSWIFTv2Middleware) GetSWIFTv2IPConfig(_ context.Context, podInfo cn } podIPInfo.MACAddress = mtpnc.Status.MacAddress podIPInfo.NICType = cns.NICTypeSecondary - podIPInfo.IsDefaultInterface = true + podIPInfo.SkipDefaultRoutes = true defaultRoute := cns.Route{ IPAddress: mtpnc.Status.PrimaryIP, diff --git a/cns/middlewares/swiftV2.go b/cns/middlewares/swiftV2.go index f522f1550f..34cd74cf17 100644 --- a/cns/middlewares/swiftV2.go +++ b/cns/middlewares/swiftV2.go @@ -62,7 +62,7 @@ func (m *SWIFTv2Middleware) GetSWIFTv2IPConfig(ctx context.Context, podInfo cns. } podIPInfo.MACAddress = mtpnc.Status.MacAddress podIPInfo.NICType = cns.NICTypeSecondary - podIPInfo.IsDefaultInterface = true + podIPInfo.SkipDefaultRoutes = true defaultRoute := cns.Route{ IPAddress: mtpnc.Status.PrimaryIP, diff --git a/cns/middlewares/swiftV2_test.go b/cns/middlewares/swiftV2_test.go index 981bca713b..8282d3e054 100644 --- a/cns/middlewares/swiftV2_test.go +++ b/cns/middlewares/swiftV2_test.go @@ -72,7 +72,7 @@ func TestGetSWIFTv2IPConfigSuccess(t *testing.T) { ipInfo, err := middleware.GetSWIFTv2IPConfig(context.TODO(), testPod1Info) assert.Equal(t, err, nil) assert.Equal(t, ipInfo.NICType, cns.NICTypeSecondary) - assert.Equal(t, ipInfo.IsDefaultInterface, true) + assert.Equal(t, ipInfo.SkipDefaultRoutes, true) } func TestGetSWIFTv2IPConfigFailure(t *testing.T) { diff --git a/cns/restserver/ipam_test.go b/cns/restserver/ipam_test.go index 662acd1303..80e013aecb 100644 --- a/cns/restserver/ipam_test.go +++ b/cns/restserver/ipam_test.go @@ -1587,5 +1587,5 @@ func TestIPAMGetSWIFTv2IP(t *testing.T) { assert.Equal(t, SWIFTv2IP, podIPInfo[2].PodIPConfig.IPAddress) assert.Equal(t, SWIFTv2MAC, podIPInfo[2].MACAddress) assert.Equal(t, cns.NICTypeSecondary, podIPInfo[2].NICType) - assert.True(t, podIPInfo[2].IsDefaultInterface) + assert.True(t, podIPInfo[2].SkipDefaultRoutes) } From 7c4987668177f9eaa06c2d7cc8b822beabe194f3 Mon Sep 17 00:00:00 2001 From: Quang Nguyen Date: Mon, 25 Sep 2023 22:14:09 +0000 Subject: [PATCH 31/85] address comments --- cns/NetworkContainerContract.go | 30 ++++++++++++++--------------- cns/middlewares/mock/mockSWIFTv2.go | 2 +- cns/middlewares/swiftV2.go | 6 +++--- cns/middlewares/swiftV2_test.go | 4 ++-- cns/restserver/ipam.go | 7 ++++--- 5 files changed, 25 insertions(+), 24 deletions(-) diff --git a/cns/NetworkContainerContract.go b/cns/NetworkContainerContract.go index c66d07ac0e..0cf6c1dda0 100644 --- a/cns/NetworkContainerContract.go +++ b/cns/NetworkContainerContract.go @@ -181,8 +181,8 @@ type PodInfo interface { Equals(PodInfo) bool // String implements string for logging PodInfos String() string - // IsMultitenant returns true if the pod is multitenant - IsMultitenant() bool + // IsSecondaryInterfacesSet returns true if there exist a secondary interface for this pod + IsSecondaryInterfacesSet() bool } type KubernetesPodInfo struct { @@ -195,10 +195,10 @@ var _ PodInfo = (*podInfo)(nil) // podInfo implements PodInfo for multiple schemas of Key type podInfo struct { KubernetesPodInfo - PodInfraContainerID string - PodInterfaceID string - Version podInfoScheme - Multitenant bool + PodInfraContainerID string + PodInterfaceID string + Version podInfoScheme + SecondaryInterfaceSet bool } func (p podInfo) String() string { @@ -252,8 +252,8 @@ func (p *podInfo) OrchestratorContext() (json.RawMessage, error) { return jsonContext, nil } -func (p *podInfo) IsMultitenant() bool { - return p.Multitenant +func (p *podInfo) IsSecondaryInterfacesSet() bool { + return p.SecondaryInterfaceSet } // NewPodInfo returns an implementation of PodInfo that returns the passed @@ -293,7 +293,7 @@ func NewPodInfoFromIPConfigsRequest(req IPConfigsRequest) (PodInfo, error) { } p.(*podInfo).PodInfraContainerID = req.InfraContainerID p.(*podInfo).PodInterfaceID = req.PodInterfaceID - p.(*podInfo).Multitenant = req.Multitenant + p.(*podInfo).SecondaryInterfaceSet = req.SecondaryInterfaceSet return p, nil } @@ -444,12 +444,12 @@ type IPConfigRequest struct { // Same as IPConfigRequest except that DesiredIPAddresses is passed in as a slice type IPConfigsRequest struct { - DesiredIPAddresses []string `json:"desiredIPAddresses"` - PodInterfaceID string `json:"podInterfaceID"` - InfraContainerID string `json:"infraContainerID"` - OrchestratorContext json.RawMessage `json:"orchestratorContext"` - Ifname string `json:"ifname"` // Used by delegated IPAM - Multitenant bool `json:"multitenant"` + DesiredIPAddresses []string `json:"desiredIPAddresses"` + PodInterfaceID string `json:"podInterfaceID"` + InfraContainerID string `json:"infraContainerID"` + OrchestratorContext json.RawMessage `json:"orchestratorContext"` + Ifname string `json:"ifname"` // Used by delegated IPAM + SecondaryInterfaceSet bool `json:"secondaryInterfaceSet"` // will be set by SWIFT v2 validator func } // IPConfigResponse is used in CNS IPAM mode as a response to CNI ADD diff --git a/cns/middlewares/mock/mockSWIFTv2.go b/cns/middlewares/mock/mockSWIFTv2.go index df5bc646d8..312a215e93 100644 --- a/cns/middlewares/mock/mockSWIFTv2.go +++ b/cns/middlewares/mock/mockSWIFTv2.go @@ -57,7 +57,7 @@ func (m *MockSWIFTv2Middleware) ValidateMultitenantIPConfigsRequest(req *cns.IPC } // check the pod labels for Swift V2, enrich the request with the multitenant flag. TBD on the label if _, ok := pod.Labels[configuration.LabelSwiftV2]; ok { - req.Multitenant = true + req.SecondaryInterfaceSet = true } return types.Success, "" } diff --git a/cns/middlewares/swiftV2.go b/cns/middlewares/swiftV2.go index 34cd74cf17..ae650b96d4 100644 --- a/cns/middlewares/swiftV2.go +++ b/cns/middlewares/swiftV2.go @@ -20,7 +20,7 @@ type SWIFTv2Middleware struct { Cli client.Client } -// validateMultitenantIPConfigsRequest validates if pod is multitenant +// validateMultitenantIPConfigsRequest validates if pod is multitenant by checking the pod labels, used in SWIFT V2 scenario. // nolint func (m *SWIFTv2Middleware) ValidateMultitenantIPConfigsRequest(req *cns.IPConfigsRequest) (respCode types.ResponseCode, message string) { // Retrieve the pod from the cluster @@ -36,9 +36,9 @@ func (m *SWIFTv2Middleware) ValidateMultitenantIPConfigsRequest(req *cns.IPConfi return types.UnexpectedError, errBuf } - // check the pod labels for Swift V2, enrich the request with the multitenant flag. TBD on the label + // check the pod labels for Swift V2, set the request's SecondaryInterfaceSet flag to true. if _, ok := pod.Labels[configuration.LabelSwiftV2]; ok { - req.Multitenant = true + req.SecondaryInterfaceSet = true } return types.Success, "" } diff --git a/cns/middlewares/swiftV2_test.go b/cns/middlewares/swiftV2_test.go index 8282d3e054..7b0a930c2f 100644 --- a/cns/middlewares/swiftV2_test.go +++ b/cns/middlewares/swiftV2_test.go @@ -32,12 +32,12 @@ func TestValidateMultitenantIPConfigsRequestSuccess(t *testing.T) { } b, _ := testPod1Info.OrchestratorContext() happyReq.OrchestratorContext = b - happyReq.Multitenant = false + happyReq.SecondaryInterfaceSet = false respCode, err := middleware.ValidateMultitenantIPConfigsRequest(happyReq) assert.Equal(t, err, "") assert.Equal(t, respCode, types.Success) - assert.Equal(t, happyReq.Multitenant, true) + assert.Equal(t, happyReq.SecondaryInterfaceSet, true) } func TestValidateMultitenantIPConfigsRequestFailure(t *testing.T) { diff --git a/cns/restserver/ipam.go b/cns/restserver/ipam.go index b2a55b476b..af376139b8 100644 --- a/cns/restserver/ipam.go +++ b/cns/restserver/ipam.go @@ -26,6 +26,7 @@ var ( // requestIPConfigHandlerHelper validates the request, assigns IPs, and returns a response func (service *HTTPRestService) requestIPConfigHandlerHelper(ctx context.Context, ipconfigsRequest cns.IPConfigsRequest) (*cns.IPConfigsResponse, error) { + // For SWIFT v2 scenario, the validator function will also modify the ipconfigsRequest. podInfo, returnCode, returnMessage := service.validateIPConfigsRequest(ipconfigsRequest) if returnCode != types.Success { return &cns.IPConfigsResponse{ @@ -38,9 +39,9 @@ func (service *HTTPRestService) requestIPConfigHandlerHelper(ctx context.Context var MTpodIPInfo cns.PodIpInfo - // Request is for multitenant pod - if podInfo.IsMultitenant() { - // If pod is multitenant and we failed to grab its MTPNC IP config, return error immediately + // Request is for pod with secondary interface(s) + if podInfo.IsSecondaryInterfacesSet() { + // If a secondary interface(s) exists for this pod and we failed to grab its MTPNC IP config, return error immediately podIPInfo, err := service.MultitenantMiddleware.GetSWIFTv2IPConfig(ctx, podInfo) if err != nil { return &cns.IPConfigsResponse{ From 33c25f6519e1d278b2547d17aea8bda7a5187d3d Mon Sep 17 00:00:00 2001 From: Quang Nguyen Date: Wed, 27 Sep 2023 21:15:21 +0000 Subject: [PATCH 32/85] refractor: make changes according to cni/cns contract --- cns/NetworkContainerContract.go | 8 -------- cns/middlewares/mock/mockSWIFTv2.go | 4 ++-- cns/middlewares/swiftV2.go | 5 +++-- cns/middlewares/swiftV2_test.go | 2 +- cns/restserver/ipam_test.go | 4 ++-- 5 files changed, 8 insertions(+), 15 deletions(-) diff --git a/cns/NetworkContainerContract.go b/cns/NetworkContainerContract.go index 830d213d1f..9295903284 100644 --- a/cns/NetworkContainerContract.go +++ b/cns/NetworkContainerContract.go @@ -87,14 +87,6 @@ const ( MultiTenantCRD = "MultiTenantCRD" ) -type NICType string - -// NIC Types -const ( - NICTypeDefault NICType = "Default" - NICTypeSecondary NICType = "Secondary" -) - // CreateNetworkContainerRequest specifies request to create a network container or network isolation boundary. type CreateNetworkContainerRequest struct { HostPrimaryIP string diff --git a/cns/middlewares/mock/mockSWIFTv2.go b/cns/middlewares/mock/mockSWIFTv2.go index 312a215e93..33adfe2c65 100644 --- a/cns/middlewares/mock/mockSWIFTv2.go +++ b/cns/middlewares/mock/mockSWIFTv2.go @@ -80,8 +80,8 @@ func (m *MockSWIFTv2Middleware) GetSWIFTv2IPConfig(_ context.Context, podInfo cn podIPInfo.PodIPConfig = cns.IPSubnet{ IPAddress: mtpnc.Status.PrimaryIP, } - podIPInfo.MACAddress = mtpnc.Status.MacAddress - podIPInfo.NICType = cns.NICTypeSecondary + podIPInfo.MacAddress = mtpnc.Status.MacAddress + podIPInfo.NICType = cns.DelegatedVMNIC podIPInfo.SkipDefaultRoutes = true defaultRoute := cns.Route{ diff --git a/cns/middlewares/swiftV2.go b/cns/middlewares/swiftV2.go index ae650b96d4..cac2513f88 100644 --- a/cns/middlewares/swiftV2.go +++ b/cns/middlewares/swiftV2.go @@ -60,9 +60,10 @@ func (m *SWIFTv2Middleware) GetSWIFTv2IPConfig(ctx context.Context, podInfo cns. podIPInfo.PodIPConfig = cns.IPSubnet{ IPAddress: mtpnc.Status.PrimaryIP, } - podIPInfo.MACAddress = mtpnc.Status.MacAddress - podIPInfo.NICType = cns.NICTypeSecondary + podIPInfo.MacAddress = mtpnc.Status.MacAddress + podIPInfo.NICType = cns.DelegatedVMNIC podIPInfo.SkipDefaultRoutes = true + // podIPInfo.InterfaceName is empty for DelegatedVMNIC defaultRoute := cns.Route{ IPAddress: mtpnc.Status.PrimaryIP, diff --git a/cns/middlewares/swiftV2_test.go b/cns/middlewares/swiftV2_test.go index 7b0a930c2f..671c7de082 100644 --- a/cns/middlewares/swiftV2_test.go +++ b/cns/middlewares/swiftV2_test.go @@ -71,7 +71,7 @@ func TestGetSWIFTv2IPConfigSuccess(t *testing.T) { ipInfo, err := middleware.GetSWIFTv2IPConfig(context.TODO(), testPod1Info) assert.Equal(t, err, nil) - assert.Equal(t, ipInfo.NICType, cns.NICTypeSecondary) + assert.Equal(t, ipInfo.NICType, cns.DelegatedVMNIC) assert.Equal(t, ipInfo.SkipDefaultRoutes, true) } diff --git a/cns/restserver/ipam_test.go b/cns/restserver/ipam_test.go index 80e013aecb..e45c944d3e 100644 --- a/cns/restserver/ipam_test.go +++ b/cns/restserver/ipam_test.go @@ -1585,7 +1585,7 @@ func TestIPAMGetSWIFTv2IP(t *testing.T) { // Asserting that multitenant IP is returned assert.Equal(t, SWIFTv2IP, podIPInfo[2].PodIPConfig.IPAddress) - assert.Equal(t, SWIFTv2MAC, podIPInfo[2].MACAddress) - assert.Equal(t, cns.NICTypeSecondary, podIPInfo[2].NICType) + assert.Equal(t, SWIFTv2MAC, podIPInfo[2].MacAddress) + assert.Equal(t, cns.DelegatedVMNIC, podIPInfo[2].NICType) assert.True(t, podIPInfo[2].SkipDefaultRoutes) } From 0972ce44f7aa28775d2461cfb61bf6301e728a5a Mon Sep 17 00:00:00 2001 From: Quang Nguyen Date: Fri, 29 Sep 2023 20:40:45 +0000 Subject: [PATCH 33/85] refractor: make adding route its own func + move swift v2 ipam branching to after normal ipam flow --- cns/api.go | 5 +- cns/configuration/env.go | 25 +++++--- cns/configuration/env_test.go | 20 +++++++ cns/middlewares/mock/mockSWIFTv2.go | 26 +++------ cns/middlewares/swiftV2.go | 88 ++++++++++++++++++----------- cns/middlewares/swiftV2_test.go | 51 ++++++++++++++--- cns/restserver/ipam.go | 52 ++++++++--------- cns/restserver/ipam_test.go | 2 +- cns/restserver/restserver.go | 6 +- 9 files changed, 175 insertions(+), 100 deletions(-) diff --git a/cns/api.go b/cns/api.go index 7a4f8c75e1..a4bcda81eb 100644 --- a/cns/api.go +++ b/cns/api.go @@ -53,8 +53,9 @@ type HTTPService interface { // Middleware interface for testing later on type SWIFTv2Middleware interface { - ValidateMultitenantIPConfigsRequest(*IPConfigsRequest) (types.ResponseCode, string) - GetSWIFTv2IPConfig(context.Context, PodInfo) (PodIpInfo, error) + ValidateIPConfigsRequest(*IPConfigsRequest) (types.ResponseCode, string) + GetIPConfig(context.Context, PodInfo) (PodIpInfo, error) + SetRoutes(*PodIpInfo) error } type IPConfigValidator func(ipConfigsRequest *IPConfigsRequest) (types.ResponseCode, string) diff --git a/cns/configuration/env.go b/cns/configuration/env.go index 7af2705952..fd2dd2d18d 100644 --- a/cns/configuration/env.go +++ b/cns/configuration/env.go @@ -13,15 +13,16 @@ const ( EnvNodeIP = "NODE_IP" // LabelSwiftV2 is the Node label for Swift V2 LabelSwiftV2 = "kubernetes.azure.com/podnetwork-multi-tenancy-enabled" - EnvPodCIDR = "POD_CIDR" + EnvPodCIDRv4 = "POD_CIDRv4" + EnvPodCIDRv6 = "POD_CIDRv6" EnvServiceCIDR = "SERVICE_CIDR" ) // ErrNodeNameUnset indicates the the $EnvNodeName variable is unset in the environment. var ErrNodeNameUnset = errors.Errorf("must declare %s environment variable", EnvNodeName) -// ErrPodCIDRUnset indicates the the $EnvPodCIDR variable is unset in the environment. -var ErrPodCIDRUnset = errors.Errorf("must declare %s environment variable", EnvPodCIDR) +// ErrPodCIDRv4Unset indicates the the $EnvPodCIDRv4 variable is unset in the environment. +var ErrPodCIDRv4Unset = errors.Errorf("must declare %s environment variable", EnvPodCIDRv4) // ErrServiceCIDRUnset indicates the the $EnvServiceCIDR variable is unset in the environment. var ErrServiceCIDRUnset = errors.Errorf("must declare %s environment variable", EnvServiceCIDR) @@ -40,12 +41,20 @@ func NodeIP() string { return os.Getenv(EnvNodeIP) } -func PodCIDR() (string, error) { - podCIDR := os.Getenv(EnvPodCIDR) - if podCIDR == "" { - return "", ErrPodCIDRUnset +func PodCIDRv4() (string, error) { + podCIDRv4 := os.Getenv(EnvPodCIDRv4) + if podCIDRv4 == "" { + return "", ErrPodCIDRv4Unset } - return podCIDR, nil + return podCIDRv4, nil +} + +func PodCIDRv6() (string, error) { + podCIDRv6 := os.Getenv(EnvPodCIDRv6) + if podCIDRv6 == "" { + return "", ErrPodCIDRv4Unset + } + return podCIDRv6, nil } func ServiceCIDR() (string, error) { diff --git a/cns/configuration/env_test.go b/cns/configuration/env_test.go index df06b6d610..2c50987d48 100644 --- a/cns/configuration/env_test.go +++ b/cns/configuration/env_test.go @@ -17,3 +17,23 @@ func TestNodeName(t *testing.T) { assert.NoError(t, err) assert.Equal(t, "test", name) } + +func TestPodCIDRv4(t *testing.T) { + _, err := PodCIDRv4() + require.Error(t, err) + require.ErrorIs(t, err, ErrPodCIDRv4Unset) + os.Setenv(EnvPodCIDRv4, "test") + cidr, err := PodCIDRv4() + assert.NoError(t, err) + assert.Equal(t, "test", cidr) +} + +func TestServiceCIDR(t *testing.T) { + _, err := ServiceCIDR() + require.Error(t, err) + require.ErrorIs(t, err, ErrServiceCIDRUnset) + os.Setenv(EnvServiceCIDR, "test") + cidr, err := ServiceCIDR() + assert.NoError(t, err) + assert.Equal(t, "test", cidr) +} diff --git a/cns/middlewares/mock/mockSWIFTv2.go b/cns/middlewares/mock/mockSWIFTv2.go index 33adfe2c65..0f12089c2d 100644 --- a/cns/middlewares/mock/mockSWIFTv2.go +++ b/cns/middlewares/mock/mockSWIFTv2.go @@ -42,7 +42,7 @@ func NewMockSWIFTv2Middleware() *MockSWIFTv2Middleware { // validateMultitenantIPConfigsRequest validates if pod is multitenant // nolint -func (m *MockSWIFTv2Middleware) ValidateMultitenantIPConfigsRequest(req *cns.IPConfigsRequest) (respCode types.ResponseCode, message string) { +func (m *MockSWIFTv2Middleware) ValidateIPConfigsRequest(req *cns.IPConfigsRequest) (respCode types.ResponseCode, message string) { // Retrieve the pod from the cluster podInfo, err := cns.UnmarshalPodInfo(req.OrchestratorContext) if err != nil { @@ -64,7 +64,7 @@ func (m *MockSWIFTv2Middleware) ValidateMultitenantIPConfigsRequest(req *cns.IPC // GetSWIFTv2IPConfig(podInfo PodInfo) (*PodIpInfo, error) // GetMultitenantIPConfig returns the IP config for a multitenant pod from the MTPNC CRD -func (m *MockSWIFTv2Middleware) GetSWIFTv2IPConfig(_ context.Context, podInfo cns.PodInfo) (cns.PodIpInfo, error) { +func (m *MockSWIFTv2Middleware) GetIPConfig(_ context.Context, podInfo cns.PodInfo) (cns.PodIpInfo, error) { // Check if the MTPNC CRD exists for the pod, if not, return error mtpncNamespacedName := k8types.NamespacedName{Namespace: podInfo.Namespace(), Name: podInfo.Name()} mtpnc, ok := m.mtpncState[mtpncNamespacedName.String()] @@ -82,23 +82,11 @@ func (m *MockSWIFTv2Middleware) GetSWIFTv2IPConfig(_ context.Context, podInfo cn } podIPInfo.MacAddress = mtpnc.Status.MacAddress podIPInfo.NICType = cns.DelegatedVMNIC - podIPInfo.SkipDefaultRoutes = true - - defaultRoute := cns.Route{ - IPAddress: mtpnc.Status.PrimaryIP, - GatewayIPAddress: mtpnc.Status.GatewayIP, - InterfaceToUse: "eth1", - } - podCIDRRoute := cns.Route{ - IPAddress: "10.0.1.10/24", - InterfaceToUse: "eth0", - } - - serviceCIDRRoute := cns.Route{ - IPAddress: "10.0.2.10/24", - InterfaceToUse: "eth0", - } - podIPInfo.Routes = []cns.Route{defaultRoute, podCIDRRoute, serviceCIDRRoute} + podIPInfo.SkipDefaultRoutes = false return podIPInfo, nil } + +func (m *MockSWIFTv2Middleware) SetRoutes(_ *cns.PodIpInfo) error { + return nil +} diff --git a/cns/middlewares/swiftV2.go b/cns/middlewares/swiftV2.go index cac2513f88..8d8cc0a029 100644 --- a/cns/middlewares/swiftV2.go +++ b/cns/middlewares/swiftV2.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "net" "github.com/Azure/azure-container-networking/cns" "github.com/Azure/azure-container-networking/cns/configuration" @@ -14,15 +15,24 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) -var ErrMTPNCNotReady = errors.New("mtpnc is not ready") +var ( + ErrMTPNCNotReady = errors.New("mtpnc is not ready") + ErrInvalidSWIFTv2NICType = errors.New("invalid NIC type for SWIFT v2 scenario") +) + +const ( + prefixLength = 32 + overlayGatewayv4 = "169.254.1.1" + overlayGatewayV6 = "fe80::1234:5678:9abc" +) type SWIFTv2Middleware struct { Cli client.Client } -// validateMultitenantIPConfigsRequest validates if pod is multitenant by checking the pod labels, used in SWIFT V2 scenario. +// ValidateIPConfigsRequest validates if pod is multitenant by checking the pod labels, used in SWIFT V2 scenario. // nolint -func (m *SWIFTv2Middleware) ValidateMultitenantIPConfigsRequest(req *cns.IPConfigsRequest) (respCode types.ResponseCode, message string) { +func (m *SWIFTv2Middleware) ValidateIPConfigsRequest(req *cns.IPConfigsRequest) (respCode types.ResponseCode, message string) { // Retrieve the pod from the cluster podInfo, err := cns.UnmarshalPodInfo(req.OrchestratorContext) if err != nil { @@ -43,8 +53,8 @@ func (m *SWIFTv2Middleware) ValidateMultitenantIPConfigsRequest(req *cns.IPConfi return types.Success, "" } -// GetMultitenantIPConfig returns the IP config for a multitenant pod from the MTPNC CRD -func (m *SWIFTv2Middleware) GetSWIFTv2IPConfig(ctx context.Context, podInfo cns.PodInfo) (cns.PodIpInfo, error) { +// GetIPConfig returns the pod's IP configuration, used in SWIFT V2 scenario. +func (m *SWIFTv2Middleware) GetIPConfig(ctx context.Context, podInfo cns.PodInfo) (cns.PodIpInfo, error) { // Check if the MTPNC CRD exists for the pod, if not, return error mtpnc := v1alpha1.MultitenantPodNetworkConfig{} mtpncNamespacedName := k8stypes.NamespacedName{Namespace: podInfo.Namespace(), Name: podInfo.Name()} @@ -58,39 +68,51 @@ func (m *SWIFTv2Middleware) GetSWIFTv2IPConfig(ctx context.Context, podInfo cns. } podIPInfo := cns.PodIpInfo{} podIPInfo.PodIPConfig = cns.IPSubnet{ - IPAddress: mtpnc.Status.PrimaryIP, + IPAddress: mtpnc.Status.PrimaryIP, + PrefixLength: prefixLength, } podIPInfo.MacAddress = mtpnc.Status.MacAddress podIPInfo.NICType = cns.DelegatedVMNIC - podIPInfo.SkipDefaultRoutes = true + podIPInfo.SkipDefaultRoutes = false // podIPInfo.InterfaceName is empty for DelegatedVMNIC - defaultRoute := cns.Route{ - IPAddress: mtpnc.Status.PrimaryIP, - GatewayIPAddress: mtpnc.Status.GatewayIP, - InterfaceToUse: "eth1", - } - - podCIDR, err := configuration.PodCIDR() - if err != nil { - return cns.PodIpInfo{}, fmt.Errorf("failed to get pod CIDR from environment : %w", err) - } - - podCIDRRoute := cns.Route{ - IPAddress: podCIDR, - InterfaceToUse: "eth0", - } - - serviceCIDR, err := configuration.ServiceCIDR() - if err != nil { - return cns.PodIpInfo{}, fmt.Errorf("failed to get service CIDR from environment : %w", err) - } + return podIPInfo, nil +} - serviceCIDRRoute := cns.Route{ - IPAddress: serviceCIDR, - InterfaceToUse: "eth0", +func (m *SWIFTv2Middleware) SetRoutes(podIPInfo *cns.PodIpInfo) error { + switch podIPInfo.NICType { + case cns.DelegatedVMNIC: + route := cns.Route{ + IPAddress: "0.0.0.0/0", + } + podIPInfo.Routes = []cns.Route{route} + case cns.InfraNIC: + // Check if IP is v4 or v6 + if net.ParseIP(podIPInfo.PodIPConfig.IPAddress).To4() != nil { + // IPv4 + podCIDRv4, err := configuration.PodCIDRv4() + if err != nil { + return fmt.Errorf("failed to get podCIDRv4 from env : %w", err) + } + podCIDRv4Route := cns.Route{ + IPAddress: podCIDRv4, + GatewayIPAddress: overlayGatewayv4, + } + podIPInfo.Routes = []cns.Route{podCIDRv4Route} + } else { + // IPv6 + podCIDRv6, err := configuration.PodCIDRv6() + if err != nil { + return fmt.Errorf("failed to get podCIDRv6 from env : %w", err) + } + podCIDRv6Route := cns.Route{ + IPAddress: podCIDRv6, + GatewayIPAddress: overlayGatewayV6, + } + podIPInfo.Routes = []cns.Route{podCIDRv6Route} + } + default: + return ErrInvalidSWIFTv2NICType } - podIPInfo.Routes = []cns.Route{defaultRoute, podCIDRRoute, serviceCIDRRoute} - - return podIPInfo, nil + return nil } diff --git a/cns/middlewares/swiftV2_test.go b/cns/middlewares/swiftV2_test.go index 671c7de082..ff469883bf 100644 --- a/cns/middlewares/swiftV2_test.go +++ b/cns/middlewares/swiftV2_test.go @@ -34,7 +34,7 @@ func TestValidateMultitenantIPConfigsRequestSuccess(t *testing.T) { happyReq.OrchestratorContext = b happyReq.SecondaryInterfaceSet = false - respCode, err := middleware.ValidateMultitenantIPConfigsRequest(happyReq) + respCode, err := middleware.ValidateIPConfigsRequest(happyReq) assert.Equal(t, err, "") assert.Equal(t, respCode, types.Success) assert.Equal(t, happyReq.SecondaryInterfaceSet, true) @@ -49,7 +49,7 @@ func TestValidateMultitenantIPConfigsRequestFailure(t *testing.T) { InfraContainerID: testPod1Info.InfraContainerID(), } failReq.OrchestratorContext = []byte("invalid") - respCode, _ := middleware.ValidateMultitenantIPConfigsRequest(failReq) + respCode, _ := middleware.ValidateIPConfigsRequest(failReq) assert.Equal(t, respCode, types.UnexpectedError) // Pod doesn't exist in cache test @@ -59,30 +59,65 @@ func TestValidateMultitenantIPConfigsRequestFailure(t *testing.T) { } b, _ := testPod2Info.OrchestratorContext() failReq.OrchestratorContext = b - respCode, _ = middleware.ValidateMultitenantIPConfigsRequest(failReq) + respCode, _ = middleware.ValidateIPConfigsRequest(failReq) assert.Equal(t, respCode, types.UnexpectedError) } func TestGetSWIFTv2IPConfigSuccess(t *testing.T) { - os.Setenv(configuration.EnvPodCIDR, "10.0.1.10/24") + os.Setenv(configuration.EnvPodCIDRv4, "10.0.1.10/24") os.Setenv(configuration.EnvServiceCIDR, "10.0.2.10/24") middleware := SWIFTv2Middleware{Cli: mock.NewMockClient()} - ipInfo, err := middleware.GetSWIFTv2IPConfig(context.TODO(), testPod1Info) + ipInfo, err := middleware.GetIPConfig(context.TODO(), testPod1Info) assert.Equal(t, err, nil) assert.Equal(t, ipInfo.NICType, cns.DelegatedVMNIC) - assert.Equal(t, ipInfo.SkipDefaultRoutes, true) + assert.Equal(t, ipInfo.SkipDefaultRoutes, false) } func TestGetSWIFTv2IPConfigFailure(t *testing.T) { middleware := SWIFTv2Middleware{Cli: mock.NewMockClient()} // Pod's MTPNC doesn't exist in cache test - _, err := middleware.GetSWIFTv2IPConfig(context.TODO(), testPod2Info) + _, err := middleware.GetIPConfig(context.TODO(), testPod2Info) assert.Error(t, err, "failed to get pod's mtpnc from cache : mtpnc not found") // Pod's MTPNC is not ready test - _, err = middleware.GetSWIFTv2IPConfig(context.TODO(), testPod3Info) + _, err = middleware.GetIPConfig(context.TODO(), testPod3Info) assert.Error(t, err, ErrMTPNCNotReady.Error()) } + +func TestSetRoutesSuccess(t *testing.T) { + middleware := SWIFTv2Middleware{Cli: mock.NewMockClient()} + os.Setenv(configuration.EnvPodCIDRv4, "10.0.1.10/24") + os.Setenv(configuration.EnvPodCIDRv6, "16A0:0010:AB00:001E::2/32") + podIPInfo := []cns.PodIpInfo{ + { + PodIPConfig: cns.IPSubnet{ + IPAddress: "10.0.1.10", + PrefixLength: 32, + }, + NICType: cns.InfraNIC, + }, + { + PodIPConfig: cns.IPSubnet{ + IPAddress: "2001:0db8:abcd:0015::0", + PrefixLength: 64, + }, + NICType: cns.InfraNIC, + }, + { + PodIPConfig: cns.IPSubnet{ + IPAddress: "20.240.1.242", + PrefixLength: 32, + }, + NICType: cns.DelegatedVMNIC, + MacAddress: "12:34:56:78:9a:bc", + }, + } + for i := range podIPInfo { + ipInfo := podIPInfo[i] + err := middleware.SetRoutes(&ipInfo) + assert.Equal(t, err, nil) + } +} diff --git a/cns/restserver/ipam.go b/cns/restserver/ipam.go index af376139b8..1c2db316c5 100644 --- a/cns/restserver/ipam.go +++ b/cns/restserver/ipam.go @@ -37,24 +37,6 @@ func (service *HTTPRestService) requestIPConfigHandlerHelper(ctx context.Context }, errors.New("failed to validate ip config request") } - var MTpodIPInfo cns.PodIpInfo - - // Request is for pod with secondary interface(s) - if podInfo.IsSecondaryInterfacesSet() { - // If a secondary interface(s) exists for this pod and we failed to grab its MTPNC IP config, return error immediately - podIPInfo, err := service.MultitenantMiddleware.GetSWIFTv2IPConfig(ctx, podInfo) - if err != nil { - return &cns.IPConfigsResponse{ - Response: cns.Response{ - ReturnCode: types.FailedToAllocateIPConfig, - Message: fmt.Sprintf("AllocateIPConfig failed: %v, IP config request is %v", err, ipconfigsRequest), - }, - PodIPInfo: []cns.PodIpInfo{}, - }, errors.Wrapf(err, "failed to get multitenant IP config %v", ipconfigsRequest) - } - MTpodIPInfo = podIPInfo - } - // record a pod requesting an IP service.podsPendingIPAssignment.Push(podInfo.Key()) @@ -91,16 +73,34 @@ func (service *HTTPRestService) requestIPConfigHandlerHelper(ctx context.Context } } - // Adding MTNpodIPInfo to the response, if it is not empty - if MTpodIPInfo.PodIPConfig.IPAddress != "" { - defaultGateway := podIPInfo[0].NetworkContainerPrimaryIPConfig.GatewayIPAddress - // Populate the non-multitenant routes with the default gateway - for i := range MTpodIPInfo.Routes { - if MTpodIPInfo.Routes[i].GatewayIPAddress == "" { - MTpodIPInfo.Routes[i].GatewayIPAddress = defaultGateway + // Check if request is for pod with secondary interface(s) + if podInfo.IsSecondaryInterfacesSet() { + // In the future, if we have multiple scenario with secondary interfaces, we can add a switch case here + SWIFTv2PodIPInfo, err := service.SWIFTv2Middleware.GetIPConfig(ctx, podInfo) + if err != nil { + return &cns.IPConfigsResponse{ + Response: cns.Response{ + ReturnCode: types.FailedToAllocateIPConfig, + Message: fmt.Sprintf("AllocateIPConfig failed: %v, IP config request is %v", err, ipconfigsRequest), + }, + PodIPInfo: []cns.PodIpInfo{}, + }, errors.Wrapf(err, "failed to get SWIFTv2 IP config %v", ipconfigsRequest) + } + podIPInfo = append(podIPInfo, SWIFTv2PodIPInfo) + // Setting up routes for SWIFTv2 scenario + for i := range podIPInfo { + ipInfo := &podIPInfo[i] + err := service.SWIFTv2Middleware.SetRoutes(ipInfo) + if err != nil { + return &cns.IPConfigsResponse{ + Response: cns.Response{ + ReturnCode: types.FailedToAllocateIPConfig, + Message: fmt.Sprintf("AllocateIPConfig failed: %v, IP config request is %v", err, ipconfigsRequest), + }, + PodIPInfo: []cns.PodIpInfo{}, + }, errors.Wrapf(err, "failed to set SWIFTv2 routes %v", ipconfigsRequest) } } - podIPInfo = append(podIPInfo, MTpodIPInfo) } return &cns.IPConfigsResponse{ diff --git a/cns/restserver/ipam_test.go b/cns/restserver/ipam_test.go index e45c944d3e..72d6e0150c 100644 --- a/cns/restserver/ipam_test.go +++ b/cns/restserver/ipam_test.go @@ -1587,5 +1587,5 @@ func TestIPAMGetSWIFTv2IP(t *testing.T) { assert.Equal(t, SWIFTv2IP, podIPInfo[2].PodIPConfig.IPAddress) assert.Equal(t, SWIFTv2MAC, podIPInfo[2].MacAddress) assert.Equal(t, cns.DelegatedVMNIC, podIPInfo[2].NICType) - assert.True(t, podIPInfo[2].SkipDefaultRoutes) + assert.False(t, podIPInfo[2].SkipDefaultRoutes) } diff --git a/cns/restserver/restserver.go b/cns/restserver/restserver.go index 7d70f9f309..1bba18bcfc 100644 --- a/cns/restserver/restserver.go +++ b/cns/restserver/restserver.go @@ -73,7 +73,7 @@ type HTTPRestService struct { cniConflistGenerator CNIConflistGenerator generateCNIConflistOnce sync.Once ipConfigsValidators []cns.IPConfigValidator - MultitenantMiddleware cns.SWIFTv2Middleware + SWIFTv2Middleware cns.SWIFTv2Middleware } type CNIConflistGenerator interface { @@ -354,7 +354,7 @@ func (service *HTTPRestService) MustGenerateCNIConflistOnce() { } func (service *HTTPRestService) AttachSWIFTv2Middleware(middleware cns.SWIFTv2Middleware) { - service.MultitenantMiddleware = middleware + service.SWIFTv2Middleware = middleware // add multitenant ipconfig validator function - service.ipConfigsValidators = append(service.ipConfigsValidators, middleware.ValidateMultitenantIPConfigsRequest) + service.ipConfigsValidators = append(service.ipConfigsValidators, middleware.ValidateIPConfigsRequest) } From 3fc14bd226378b7b0a15e3b34756987ff591ec51 Mon Sep 17 00:00:00 2001 From: Quang Nguyen Date: Fri, 29 Sep 2023 20:45:59 +0000 Subject: [PATCH 34/85] refractor: change vars naming --- cns/NetworkContainerContract.go | 20 ++++++++++---------- cns/middlewares/mock/mockSWIFTv2.go | 2 +- cns/middlewares/swiftV2.go | 2 +- cns/middlewares/swiftV2_test.go | 4 ++-- cns/restserver/ipam.go | 2 +- 5 files changed, 15 insertions(+), 15 deletions(-) diff --git a/cns/NetworkContainerContract.go b/cns/NetworkContainerContract.go index 9295903284..114cb615c1 100644 --- a/cns/NetworkContainerContract.go +++ b/cns/NetworkContainerContract.go @@ -182,8 +182,8 @@ type PodInfo interface { Equals(PodInfo) bool // String implements string for logging PodInfos String() string - // IsSecondaryInterfacesSet returns true if there exist a secondary interface for this pod - IsSecondaryInterfacesSet() bool + // SecondaryInterfacesExist returns true if there exist a secondary interface for this pod + SecondaryInterfacesExist() bool } type KubernetesPodInfo struct { @@ -253,7 +253,7 @@ func (p *podInfo) OrchestratorContext() (json.RawMessage, error) { return jsonContext, nil } -func (p *podInfo) IsSecondaryInterfacesSet() bool { +func (p *podInfo) SecondaryInterfacesExist() bool { return p.SecondaryInterfaceSet } @@ -294,7 +294,7 @@ func NewPodInfoFromIPConfigsRequest(req IPConfigsRequest) (PodInfo, error) { } p.(*podInfo).PodInfraContainerID = req.InfraContainerID p.(*podInfo).PodInterfaceID = req.PodInterfaceID - p.(*podInfo).SecondaryInterfaceSet = req.SecondaryInterfaceSet + p.(*podInfo).SecondaryInterfaceSet = req.SecondaryInterfacesExist return p, nil } @@ -450,12 +450,12 @@ type IPConfigRequest struct { // Same as IPConfigRequest except that DesiredIPAddresses is passed in as a slice type IPConfigsRequest struct { - DesiredIPAddresses []string `json:"desiredIPAddresses"` - PodInterfaceID string `json:"podInterfaceID"` - InfraContainerID string `json:"infraContainerID"` - OrchestratorContext json.RawMessage `json:"orchestratorContext"` - Ifname string `json:"ifname"` // Used by delegated IPAM - SecondaryInterfaceSet bool `json:"secondaryInterfaceSet"` // will be set by SWIFT v2 validator func + DesiredIPAddresses []string `json:"desiredIPAddresses"` + PodInterfaceID string `json:"podInterfaceID"` + InfraContainerID string `json:"infraContainerID"` + OrchestratorContext json.RawMessage `json:"orchestratorContext"` + Ifname string `json:"ifname"` // Used by delegated IPAM + SecondaryInterfacesExist bool `json:"secondaryInterfacesExist"` // will be set by SWIFT v2 validator func } // IPConfigResponse is used in CNS IPAM mode as a response to CNI ADD diff --git a/cns/middlewares/mock/mockSWIFTv2.go b/cns/middlewares/mock/mockSWIFTv2.go index 0f12089c2d..8c0ad98526 100644 --- a/cns/middlewares/mock/mockSWIFTv2.go +++ b/cns/middlewares/mock/mockSWIFTv2.go @@ -57,7 +57,7 @@ func (m *MockSWIFTv2Middleware) ValidateIPConfigsRequest(req *cns.IPConfigsReque } // check the pod labels for Swift V2, enrich the request with the multitenant flag. TBD on the label if _, ok := pod.Labels[configuration.LabelSwiftV2]; ok { - req.SecondaryInterfaceSet = true + req.SecondaryInterfacesExist = true } return types.Success, "" } diff --git a/cns/middlewares/swiftV2.go b/cns/middlewares/swiftV2.go index 8d8cc0a029..09265a64fd 100644 --- a/cns/middlewares/swiftV2.go +++ b/cns/middlewares/swiftV2.go @@ -48,7 +48,7 @@ func (m *SWIFTv2Middleware) ValidateIPConfigsRequest(req *cns.IPConfigsRequest) // check the pod labels for Swift V2, set the request's SecondaryInterfaceSet flag to true. if _, ok := pod.Labels[configuration.LabelSwiftV2]; ok { - req.SecondaryInterfaceSet = true + req.SecondaryInterfacesExist = true } return types.Success, "" } diff --git a/cns/middlewares/swiftV2_test.go b/cns/middlewares/swiftV2_test.go index ff469883bf..b728813051 100644 --- a/cns/middlewares/swiftV2_test.go +++ b/cns/middlewares/swiftV2_test.go @@ -32,12 +32,12 @@ func TestValidateMultitenantIPConfigsRequestSuccess(t *testing.T) { } b, _ := testPod1Info.OrchestratorContext() happyReq.OrchestratorContext = b - happyReq.SecondaryInterfaceSet = false + happyReq.SecondaryInterfacesExist = false respCode, err := middleware.ValidateIPConfigsRequest(happyReq) assert.Equal(t, err, "") assert.Equal(t, respCode, types.Success) - assert.Equal(t, happyReq.SecondaryInterfaceSet, true) + assert.Equal(t, happyReq.SecondaryInterfacesExist, true) } func TestValidateMultitenantIPConfigsRequestFailure(t *testing.T) { diff --git a/cns/restserver/ipam.go b/cns/restserver/ipam.go index 1c2db316c5..f687cf369a 100644 --- a/cns/restserver/ipam.go +++ b/cns/restserver/ipam.go @@ -74,7 +74,7 @@ func (service *HTTPRestService) requestIPConfigHandlerHelper(ctx context.Context } // Check if request is for pod with secondary interface(s) - if podInfo.IsSecondaryInterfacesSet() { + if podInfo.SecondaryInterfacesExist() { // In the future, if we have multiple scenario with secondary interfaces, we can add a switch case here SWIFTv2PodIPInfo, err := service.SWIFTv2Middleware.GetIPConfig(ctx, podInfo) if err != nil { From 28ba7b20e7b44c07dd6e347df470d2a39537a25d Mon Sep 17 00:00:00 2001 From: Quang Nguyen Date: Fri, 29 Sep 2023 20:51:30 +0000 Subject: [PATCH 35/85] refractor: more var naming --- cns/configuration/env.go | 23 +++++++++++++---------- cns/configuration/env_test.go | 8 ++++---- cns/middlewares/swiftV2.go | 4 ++-- cns/middlewares/swiftV2_test.go | 6 +++--- 4 files changed, 22 insertions(+), 19 deletions(-) diff --git a/cns/configuration/env.go b/cns/configuration/env.go index fd2dd2d18d..3dbdc6d50a 100644 --- a/cns/configuration/env.go +++ b/cns/configuration/env.go @@ -13,16 +13,19 @@ const ( EnvNodeIP = "NODE_IP" // LabelSwiftV2 is the Node label for Swift V2 LabelSwiftV2 = "kubernetes.azure.com/podnetwork-multi-tenancy-enabled" - EnvPodCIDRv4 = "POD_CIDRv4" - EnvPodCIDRv6 = "POD_CIDRv6" + EnvPodV4CIDR = "POD_V4_CIDR" + EnvPodV6CIDR = "POD_V6_CIDR" EnvServiceCIDR = "SERVICE_CIDR" ) // ErrNodeNameUnset indicates the the $EnvNodeName variable is unset in the environment. var ErrNodeNameUnset = errors.Errorf("must declare %s environment variable", EnvNodeName) -// ErrPodCIDRv4Unset indicates the the $EnvPodCIDRv4 variable is unset in the environment. -var ErrPodCIDRv4Unset = errors.Errorf("must declare %s environment variable", EnvPodCIDRv4) +// ErrPodV4CIDRUnset indicates the the $EnvPodV4CIDR variable is unset in the environment. +var ErrPodV4CIDRUnset = errors.Errorf("must declare %s environment variable", EnvPodV4CIDR) + +// ErrPodV6CIDRUnset indicates the the $EnvPodV6CIDR variable is unset in the environment. +var ErrPodV6CIDRUnset = errors.Errorf("must declare %s environment variable", EnvPodV6CIDR) // ErrServiceCIDRUnset indicates the the $EnvServiceCIDR variable is unset in the environment. var ErrServiceCIDRUnset = errors.Errorf("must declare %s environment variable", EnvServiceCIDR) @@ -41,18 +44,18 @@ func NodeIP() string { return os.Getenv(EnvNodeIP) } -func PodCIDRv4() (string, error) { - podCIDRv4 := os.Getenv(EnvPodCIDRv4) +func PodV4CIDR() (string, error) { + podCIDRv4 := os.Getenv(EnvPodV4CIDR) if podCIDRv4 == "" { - return "", ErrPodCIDRv4Unset + return "", ErrPodV4CIDRUnset } return podCIDRv4, nil } -func PodCIDRv6() (string, error) { - podCIDRv6 := os.Getenv(EnvPodCIDRv6) +func PodV6CIDR() (string, error) { + podCIDRv6 := os.Getenv(EnvPodV6CIDR) if podCIDRv6 == "" { - return "", ErrPodCIDRv4Unset + return "", ErrPodV6CIDRUnset } return podCIDRv6, nil } diff --git a/cns/configuration/env_test.go b/cns/configuration/env_test.go index 2c50987d48..e58dcc6cab 100644 --- a/cns/configuration/env_test.go +++ b/cns/configuration/env_test.go @@ -19,11 +19,11 @@ func TestNodeName(t *testing.T) { } func TestPodCIDRv4(t *testing.T) { - _, err := PodCIDRv4() + _, err := PodV4CIDR() require.Error(t, err) - require.ErrorIs(t, err, ErrPodCIDRv4Unset) - os.Setenv(EnvPodCIDRv4, "test") - cidr, err := PodCIDRv4() + require.ErrorIs(t, err, ErrPodV4CIDRUnset) + os.Setenv(EnvPodV4CIDR, "test") + cidr, err := PodV4CIDR() assert.NoError(t, err) assert.Equal(t, "test", cidr) } diff --git a/cns/middlewares/swiftV2.go b/cns/middlewares/swiftV2.go index 09265a64fd..b133d1ad4c 100644 --- a/cns/middlewares/swiftV2.go +++ b/cns/middlewares/swiftV2.go @@ -90,7 +90,7 @@ func (m *SWIFTv2Middleware) SetRoutes(podIPInfo *cns.PodIpInfo) error { // Check if IP is v4 or v6 if net.ParseIP(podIPInfo.PodIPConfig.IPAddress).To4() != nil { // IPv4 - podCIDRv4, err := configuration.PodCIDRv4() + podCIDRv4, err := configuration.PodV4CIDR() if err != nil { return fmt.Errorf("failed to get podCIDRv4 from env : %w", err) } @@ -101,7 +101,7 @@ func (m *SWIFTv2Middleware) SetRoutes(podIPInfo *cns.PodIpInfo) error { podIPInfo.Routes = []cns.Route{podCIDRv4Route} } else { // IPv6 - podCIDRv6, err := configuration.PodCIDRv6() + podCIDRv6, err := configuration.PodV6CIDR() if err != nil { return fmt.Errorf("failed to get podCIDRv6 from env : %w", err) } diff --git a/cns/middlewares/swiftV2_test.go b/cns/middlewares/swiftV2_test.go index b728813051..aaeac148da 100644 --- a/cns/middlewares/swiftV2_test.go +++ b/cns/middlewares/swiftV2_test.go @@ -64,7 +64,7 @@ func TestValidateMultitenantIPConfigsRequestFailure(t *testing.T) { } func TestGetSWIFTv2IPConfigSuccess(t *testing.T) { - os.Setenv(configuration.EnvPodCIDRv4, "10.0.1.10/24") + os.Setenv(configuration.EnvPodV4CIDR, "10.0.1.10/24") os.Setenv(configuration.EnvServiceCIDR, "10.0.2.10/24") middleware := SWIFTv2Middleware{Cli: mock.NewMockClient()} @@ -89,8 +89,8 @@ func TestGetSWIFTv2IPConfigFailure(t *testing.T) { func TestSetRoutesSuccess(t *testing.T) { middleware := SWIFTv2Middleware{Cli: mock.NewMockClient()} - os.Setenv(configuration.EnvPodCIDRv4, "10.0.1.10/24") - os.Setenv(configuration.EnvPodCIDRv6, "16A0:0010:AB00:001E::2/32") + os.Setenv(configuration.EnvPodV4CIDR, "10.0.1.10/24") + os.Setenv(configuration.EnvPodV6CIDR, "16A0:0010:AB00:001E::2/32") podIPInfo := []cns.PodIpInfo{ { PodIPConfig: cns.IPSubnet{ From b06720040cb4f9a05e0fa2193e8dddcdf33cb72b Mon Sep 17 00:00:00 2001 From: Quang Nguyen Date: Fri, 29 Sep 2023 20:52:50 +0000 Subject: [PATCH 36/85] test: add test for podv6cidr --- cns/configuration/env_test.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/cns/configuration/env_test.go b/cns/configuration/env_test.go index e58dcc6cab..5432c467f9 100644 --- a/cns/configuration/env_test.go +++ b/cns/configuration/env_test.go @@ -18,7 +18,7 @@ func TestNodeName(t *testing.T) { assert.Equal(t, "test", name) } -func TestPodCIDRv4(t *testing.T) { +func TestPodV4CIDR(t *testing.T) { _, err := PodV4CIDR() require.Error(t, err) require.ErrorIs(t, err, ErrPodV4CIDRUnset) @@ -28,6 +28,16 @@ func TestPodCIDRv4(t *testing.T) { assert.Equal(t, "test", cidr) } +func TestPodV6CIDR(t *testing.T) { + _, err := PodV6CIDR() + require.Error(t, err) + require.ErrorIs(t, err, ErrPodV6CIDRUnset) + os.Setenv(EnvPodV6CIDR, "test") + cidr, err := PodV6CIDR() + assert.NoError(t, err) + assert.Equal(t, "test", cidr) +} + func TestServiceCIDR(t *testing.T) { _, err := ServiceCIDR() require.Error(t, err) From 4ffe8d6b216b03fdedc58e868b1f3675f207795b Mon Sep 17 00:00:00 2001 From: Quang Nguyen Date: Fri, 29 Sep 2023 21:00:57 +0000 Subject: [PATCH 37/85] refractor: make the returning podIpInfo init cleaner in swiftv2.go --- cns/middlewares/swiftV2.go | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/cns/middlewares/swiftV2.go b/cns/middlewares/swiftV2.go index b133d1ad4c..f275136b60 100644 --- a/cns/middlewares/swiftV2.go +++ b/cns/middlewares/swiftV2.go @@ -66,15 +66,16 @@ func (m *SWIFTv2Middleware) GetIPConfig(ctx context.Context, podInfo cns.PodInfo if mtpnc.Status.PrimaryIP == "" || mtpnc.Status.MacAddress == "" || mtpnc.Status.NCID == "" || mtpnc.Status.GatewayIP == "" { return cns.PodIpInfo{}, ErrMTPNCNotReady } - podIPInfo := cns.PodIpInfo{} - podIPInfo.PodIPConfig = cns.IPSubnet{ - IPAddress: mtpnc.Status.PrimaryIP, - PrefixLength: prefixLength, + podIPInfo := cns.PodIpInfo{ + PodIPConfig: cns.IPSubnet{ + IPAddress: mtpnc.Status.PrimaryIP, + PrefixLength: prefixLength, + }, + MacAddress: mtpnc.Status.MacAddress, + NICType: cns.DelegatedVMNIC, + SkipDefaultRoutes: false, + // InterfaceName is empty for DelegatedVMNIC } - podIPInfo.MacAddress = mtpnc.Status.MacAddress - podIPInfo.NICType = cns.DelegatedVMNIC - podIPInfo.SkipDefaultRoutes = false - // podIPInfo.InterfaceName is empty for DelegatedVMNIC return podIPInfo, nil } From 0a25e6532b2b02dbacebf24f0a6eb7ff5573379f Mon Sep 17 00:00:00 2001 From: Quang Nguyen Date: Mon, 2 Oct 2023 21:45:36 +0000 Subject: [PATCH 38/85] refractor + tests: add contexts to ipconfigs req validators + set route tests --- cns/api.go | 4 +-- cns/middlewares/mock/mockSWIFTv2.go | 2 +- cns/middlewares/swiftV2.go | 12 ++++--- cns/middlewares/swiftV2_test.go | 54 ++++++++++++++++++++++++++--- cns/restserver/ipam.go | 14 ++++---- cns/restserver/ipam_test.go | 2 +- cns/restserver/restserver.go | 22 ++++++------ cns/restserver/util.go | 11 +++--- 8 files changed, 83 insertions(+), 38 deletions(-) diff --git a/cns/api.go b/cns/api.go index a4bcda81eb..88bcc8045a 100644 --- a/cns/api.go +++ b/cns/api.go @@ -53,12 +53,12 @@ type HTTPService interface { // Middleware interface for testing later on type SWIFTv2Middleware interface { - ValidateIPConfigsRequest(*IPConfigsRequest) (types.ResponseCode, string) + ValidateIPConfigsRequest(context.Context, *IPConfigsRequest) (types.ResponseCode, string) GetIPConfig(context.Context, PodInfo) (PodIpInfo, error) SetRoutes(*PodIpInfo) error } -type IPConfigValidator func(ipConfigsRequest *IPConfigsRequest) (types.ResponseCode, string) +type IPConfigValidator func(context.Context, *IPConfigsRequest) (types.ResponseCode, string) // This is used for KubernetesCRD orchestrator Type where NC has multiple ips. // This struct captures the state for SecondaryIPs associated to a given NC diff --git a/cns/middlewares/mock/mockSWIFTv2.go b/cns/middlewares/mock/mockSWIFTv2.go index 8c0ad98526..ba8c31c48e 100644 --- a/cns/middlewares/mock/mockSWIFTv2.go +++ b/cns/middlewares/mock/mockSWIFTv2.go @@ -42,7 +42,7 @@ func NewMockSWIFTv2Middleware() *MockSWIFTv2Middleware { // validateMultitenantIPConfigsRequest validates if pod is multitenant // nolint -func (m *MockSWIFTv2Middleware) ValidateIPConfigsRequest(req *cns.IPConfigsRequest) (respCode types.ResponseCode, message string) { +func (m *MockSWIFTv2Middleware) ValidateIPConfigsRequest(_ context.Context, req *cns.IPConfigsRequest) (respCode types.ResponseCode, message string) { // Retrieve the pod from the cluster podInfo, err := cns.UnmarshalPodInfo(req.OrchestratorContext) if err != nil { diff --git a/cns/middlewares/swiftV2.go b/cns/middlewares/swiftV2.go index f275136b60..affceef605 100644 --- a/cns/middlewares/swiftV2.go +++ b/cns/middlewares/swiftV2.go @@ -32,7 +32,7 @@ type SWIFTv2Middleware struct { // ValidateIPConfigsRequest validates if pod is multitenant by checking the pod labels, used in SWIFT V2 scenario. // nolint -func (m *SWIFTv2Middleware) ValidateIPConfigsRequest(req *cns.IPConfigsRequest) (respCode types.ResponseCode, message string) { +func (m *SWIFTv2Middleware) ValidateIPConfigsRequest(ctx context.Context, req *cns.IPConfigsRequest) (respCode types.ResponseCode, message string) { // Retrieve the pod from the cluster podInfo, err := cns.UnmarshalPodInfo(req.OrchestratorContext) if err != nil { @@ -41,7 +41,7 @@ func (m *SWIFTv2Middleware) ValidateIPConfigsRequest(req *cns.IPConfigsRequest) } podNamespacedName := k8stypes.NamespacedName{Namespace: podInfo.Namespace(), Name: podInfo.Name()} pod := v1.Pod{} - if err := m.Cli.Get(context.TODO(), podNamespacedName, &pod); err != nil { + if err := m.Cli.Get(ctx, podNamespacedName, &pod); err != nil { errBuf := fmt.Sprintf("failed to get pod %v with error %v", podNamespacedName, err) return types.UnexpectedError, errBuf } @@ -53,7 +53,7 @@ func (m *SWIFTv2Middleware) ValidateIPConfigsRequest(req *cns.IPConfigsRequest) return types.Success, "" } -// GetIPConfig returns the pod's IP configuration, used in SWIFT V2 scenario. +// GetIPConfig returns the pod's SWIFT V2 IP configuration. func (m *SWIFTv2Middleware) GetIPConfig(ctx context.Context, podInfo cns.PodInfo) (cns.PodIpInfo, error) { // Check if the MTPNC CRD exists for the pod, if not, return error mtpnc := v1alpha1.MultitenantPodNetworkConfig{} @@ -80,9 +80,11 @@ func (m *SWIFTv2Middleware) GetIPConfig(ctx context.Context, podInfo cns.PodInfo return podIPInfo, nil } +// SetRoutes sets the routes for podIPInfo used in SWIFT V2 scenario. func (m *SWIFTv2Middleware) SetRoutes(podIPInfo *cns.PodIpInfo) error { switch podIPInfo.NICType { case cns.DelegatedVMNIC: + // default route via SWIFT v2 interface route := cns.Route{ IPAddress: "0.0.0.0/0", } @@ -90,7 +92,7 @@ func (m *SWIFTv2Middleware) SetRoutes(podIPInfo *cns.PodIpInfo) error { case cns.InfraNIC: // Check if IP is v4 or v6 if net.ParseIP(podIPInfo.PodIPConfig.IPAddress).To4() != nil { - // IPv4 + // route for IPv4 podCIDR traffic podCIDRv4, err := configuration.PodV4CIDR() if err != nil { return fmt.Errorf("failed to get podCIDRv4 from env : %w", err) @@ -101,7 +103,7 @@ func (m *SWIFTv2Middleware) SetRoutes(podIPInfo *cns.PodIpInfo) error { } podIPInfo.Routes = []cns.Route{podCIDRv4Route} } else { - // IPv6 + // route for IPv6 podCIDR traffic podCIDRv6, err := configuration.PodV6CIDR() if err != nil { return fmt.Errorf("failed to get podCIDRv6 from env : %w", err) diff --git a/cns/middlewares/swiftV2_test.go b/cns/middlewares/swiftV2_test.go index aaeac148da..0f4284d224 100644 --- a/cns/middlewares/swiftV2_test.go +++ b/cns/middlewares/swiftV2_test.go @@ -34,7 +34,7 @@ func TestValidateMultitenantIPConfigsRequestSuccess(t *testing.T) { happyReq.OrchestratorContext = b happyReq.SecondaryInterfacesExist = false - respCode, err := middleware.ValidateIPConfigsRequest(happyReq) + respCode, err := middleware.ValidateIPConfigsRequest(context.TODO(), happyReq) assert.Equal(t, err, "") assert.Equal(t, respCode, types.Success) assert.Equal(t, happyReq.SecondaryInterfacesExist, true) @@ -49,7 +49,7 @@ func TestValidateMultitenantIPConfigsRequestFailure(t *testing.T) { InfraContainerID: testPod1Info.InfraContainerID(), } failReq.OrchestratorContext = []byte("invalid") - respCode, _ := middleware.ValidateIPConfigsRequest(failReq) + respCode, _ := middleware.ValidateIPConfigsRequest(context.TODO(), failReq) assert.Equal(t, respCode, types.UnexpectedError) // Pod doesn't exist in cache test @@ -59,7 +59,7 @@ func TestValidateMultitenantIPConfigsRequestFailure(t *testing.T) { } b, _ := testPod2Info.OrchestratorContext() failReq.OrchestratorContext = b - respCode, _ = middleware.ValidateIPConfigsRequest(failReq) + respCode, _ = middleware.ValidateIPConfigsRequest(context.TODO(), failReq) assert.Equal(t, respCode, types.UnexpectedError) } @@ -115,9 +115,53 @@ func TestSetRoutesSuccess(t *testing.T) { MacAddress: "12:34:56:78:9a:bc", }, } + desiredPodIPInfo := []cns.PodIpInfo{ + { + PodIPConfig: cns.IPSubnet{ + IPAddress: "10.0.1.10", + PrefixLength: 32, + }, + NICType: cns.InfraNIC, + Routes: []cns.Route{ + { + IPAddress: "10.0.1.10/24", + GatewayIPAddress: overlayGatewayv4, + }, + }, + }, + { + PodIPConfig: cns.IPSubnet{ + IPAddress: "2001:0db8:abcd:0015::0", + PrefixLength: 64, + }, + NICType: cns.InfraNIC, + Routes: []cns.Route{ + { + IPAddress: "16A0:0010:AB00:001E::2/32", + GatewayIPAddress: overlayGatewayV6, + }, + }, + }, + { + PodIPConfig: cns.IPSubnet{ + IPAddress: "20.240.1.242", + PrefixLength: 32, + }, + NICType: cns.DelegatedVMNIC, + MacAddress: "12:34:56:78:9a:bc", + Routes: []cns.Route{ + { + IPAddress: "0.0.0.0/0", + }, + }, + }, + } for i := range podIPInfo { - ipInfo := podIPInfo[i] - err := middleware.SetRoutes(&ipInfo) + ipInfo := &podIPInfo[i] + err := middleware.SetRoutes(ipInfo) assert.Equal(t, err, nil) } + for i := range podIPInfo { + assert.DeepEqual(t, podIPInfo[i].Routes, desiredPodIPInfo[i].Routes) + } } diff --git a/cns/restserver/ipam.go b/cns/restserver/ipam.go index f687cf369a..2e51ef6bde 100644 --- a/cns/restserver/ipam.go +++ b/cns/restserver/ipam.go @@ -27,7 +27,7 @@ var ( // requestIPConfigHandlerHelper validates the request, assigns IPs, and returns a response func (service *HTTPRestService) requestIPConfigHandlerHelper(ctx context.Context, ipconfigsRequest cns.IPConfigsRequest) (*cns.IPConfigsResponse, error) { // For SWIFT v2 scenario, the validator function will also modify the ipconfigsRequest. - podInfo, returnCode, returnMessage := service.validateIPConfigsRequest(ipconfigsRequest) + podInfo, returnCode, returnMessage := service.validateIPConfigsRequest(ctx, ipconfigsRequest) if returnCode != types.Success { return &cns.IPConfigsResponse{ Response: cns.Response{ @@ -149,7 +149,7 @@ func (service *HTTPRestService) requestIPConfigHandler(w http.ResponseWriter, r } } - ipConfigsResp, errResp := service.requestIPConfigHandlerHelper(context.Background(), ipconfigsRequest) //nolint:contextcheck // appease linter + ipConfigsResp, errResp := service.requestIPConfigHandlerHelper(r.Context(), ipconfigsRequest) //nolint:contextcheck // appease linter if errResp != nil { // As this API is expected to return IPConfigResponse, generate it from the IPConfigsResponse returned above reserveResp := &cns.IPConfigResponse{ @@ -196,7 +196,7 @@ func (service *HTTPRestService) requestIPConfigsHandler(w http.ResponseWriter, r return } - ipConfigsResp, err := service.requestIPConfigHandlerHelper(context.Background(), ipconfigsRequest) // nolint:contextcheck // appease linter + ipConfigsResp, err := service.requestIPConfigHandlerHelper(r.Context(), ipconfigsRequest) // nolint:contextcheck // appease linter if err != nil { w.Header().Set(cnsReturnCode, ipConfigsResp.Response.ReturnCode.String()) err = service.Listener.Encode(w, &ipConfigsResp) @@ -272,8 +272,8 @@ func (service *HTTPRestService) updateEndpointState(ipconfigsRequest cns.IPConfi } // releaseIPConfigHandlerHelper validates the request and removes the endpoint associated with the pod -func (service *HTTPRestService) releaseIPConfigHandlerHelper(ipconfigsRequest cns.IPConfigsRequest) (*cns.Response, error) { - podInfo, returnCode, returnMessage := service.validateIPConfigsRequest(ipconfigsRequest) +func (service *HTTPRestService) releaseIPConfigHandlerHelper(ctx context.Context, ipconfigsRequest cns.IPConfigsRequest) (*cns.Response, error) { + podInfo, returnCode, returnMessage := service.validateIPConfigsRequest(ctx, ipconfigsRequest) if returnCode != types.Success { return &cns.Response{ ReturnCode: returnCode, @@ -345,7 +345,7 @@ func (service *HTTPRestService) releaseIPConfigHandler(w http.ResponseWriter, r Ifname: ipconfigRequest.Ifname, } - resp, err := service.releaseIPConfigHandlerHelper(ipconfigsRequest) + resp, err := service.releaseIPConfigHandlerHelper(r.Context(), ipconfigsRequest) if err != nil { w.Header().Set(cnsReturnCode, resp.ReturnCode.String()) err = service.Listener.Encode(w, &resp) @@ -374,7 +374,7 @@ func (service *HTTPRestService) releaseIPConfigsHandler(w http.ResponseWriter, r return } - resp, err := service.releaseIPConfigHandlerHelper(ipconfigsRequest) + resp, err := service.releaseIPConfigHandlerHelper(r.Context(), ipconfigsRequest) if err != nil { w.Header().Set(cnsReturnCode, resp.ReturnCode.String()) err = service.Listener.Encode(w, &resp) diff --git a/cns/restserver/ipam_test.go b/cns/restserver/ipam_test.go index 72d6e0150c..56ef89dcc3 100644 --- a/cns/restserver/ipam_test.go +++ b/cns/restserver/ipam_test.go @@ -67,7 +67,7 @@ func getTestService() *HTTPRestService { var config common.ServiceConfig httpsvc, _ := NewHTTPRestService(&config, &fakes.WireserverClientFake{}, &fakes.WireserverProxyFake{}, &fakes.NMAgentClientFake{}, store.NewMockStore(""), nil, nil) svc = httpsvc - svc.ipConfigsValidators = append(svc.ipConfigsValidators, svc.validateDefaultIPConfigsRequest) + svc.ipConfigsRequestValidators = append(svc.ipConfigsRequestValidators, svc.validateDefaultIPConfigsRequest) httpsvc.IPAMPoolMonitor = &fakes.MonitorFake{} setOrchestratorTypeInternal(cns.KubernetesCRD) diff --git a/cns/restserver/restserver.go b/cns/restserver/restserver.go index 1bba18bcfc..eb42b03932 100644 --- a/cns/restserver/restserver.go +++ b/cns/restserver/restserver.go @@ -67,13 +67,13 @@ type HTTPRestService struct { state *httpRestServiceState podsPendingIPAssignment *bounded.TimedSet sync.RWMutex - dncPartitionKey string - EndpointState map[string]*EndpointInfo // key : container id - EndpointStateStore store.KeyValueStore - cniConflistGenerator CNIConflistGenerator - generateCNIConflistOnce sync.Once - ipConfigsValidators []cns.IPConfigValidator - SWIFTv2Middleware cns.SWIFTv2Middleware + dncPartitionKey string + EndpointState map[string]*EndpointInfo // key : container id + EndpointStateStore store.KeyValueStore + cniConflistGenerator CNIConflistGenerator + generateCNIConflistOnce sync.Once + ipConfigsRequestValidators []cns.IPConfigValidator + SWIFTv2Middleware cns.SWIFTv2Middleware } type CNIConflistGenerator interface { @@ -228,8 +228,8 @@ func (service *HTTPRestService) Init(config *common.ServiceConfig) error { return err } - // Adding ipConfigsValidators - service.ipConfigsValidators = []cns.IPConfigValidator{service.validateDefaultIPConfigsRequest} + // Adding the default ipconfigs request validator + service.ipConfigsRequestValidators = []cns.IPConfigValidator{service.validateDefaultIPConfigsRequest} // Add handlers. listener := service.Listener @@ -355,6 +355,6 @@ func (service *HTTPRestService) MustGenerateCNIConflistOnce() { func (service *HTTPRestService) AttachSWIFTv2Middleware(middleware cns.SWIFTv2Middleware) { service.SWIFTv2Middleware = middleware - // add multitenant ipconfig validator function - service.ipConfigsValidators = append(service.ipConfigsValidators, middleware.ValidateIPConfigsRequest) + // adding the SWIFT v2 ipconfigs request validator + service.ipConfigsRequestValidators = append(service.ipConfigsRequestValidators, middleware.ValidateIPConfigsRequest) } diff --git a/cns/restserver/util.go b/cns/restserver/util.go index b5fd47d2bf..b1b83426c6 100644 --- a/cns/restserver/util.go +++ b/cns/restserver/util.go @@ -766,12 +766,10 @@ func (service *HTTPRestService) SendNCSnapShotPeriodically(ctx context.Context, } } -func (service *HTTPRestService) validateIPConfigsRequest( - ipConfigsRequest cns.IPConfigsRequest, -) (cns.PodInfo, types.ResponseCode, string) { +func (service *HTTPRestService) validateIPConfigsRequest(ctx context.Context, ipConfigsRequest cns.IPConfigsRequest) (cns.PodInfo, types.ResponseCode, string) { // looping through all the ipconfigs request validators, if any validator fails, return the error - for _, validator := range service.ipConfigsValidators { - respCode, message := validator(&ipConfigsRequest) + for _, validator := range service.ipConfigsRequestValidators { + respCode, message := validator(ctx, &ipConfigsRequest) if respCode != types.Success { return nil, respCode, message } @@ -786,7 +784,7 @@ func (service *HTTPRestService) validateIPConfigsRequest( } // validateDefaultIPConfigsRequest validates the request for default IP configs request -func (service *HTTPRestService) validateDefaultIPConfigsRequest(ipConfigsRequest *cns.IPConfigsRequest) (respCode types.ResponseCode, message string) { +func (service *HTTPRestService) validateDefaultIPConfigsRequest(_ context.Context, ipConfigsRequest *cns.IPConfigsRequest) (respCode types.ResponseCode, message string) { if service.state.OrchestratorType != cns.KubernetesCRD && service.state.OrchestratorType != cns.Kubernetes { return types.UnsupportedOrchestratorType, "ReleaseIPConfig API supported only for kubernetes orchestrator" } @@ -838,6 +836,7 @@ func (service *HTTPRestService) populateIPConfigInfoUntransacted(ipConfigStatus podIPInfo.HostPrimaryIPInfo.PrimaryIP = primaryHostInterface.PrimaryIP podIPInfo.HostPrimaryIPInfo.Subnet = primaryHostInterface.Subnet podIPInfo.HostPrimaryIPInfo.Gateway = primaryHostInterface.Gateway + podIPInfo.NICType = cns.InfraNIC return nil } From 7153f9bc9b721d23db9256ffae3b2ecf054f74ff Mon Sep 17 00:00:00 2001 From: Quang Nguyen Date: Tue, 3 Oct 2023 16:37:58 +0000 Subject: [PATCH 39/85] refractor: change labels for swift v2 pods --- cns/api.go | 2 +- cns/configuration/env.go | 11 ++++++----- cns/middlewares/mock/mockClient.go | 2 +- cns/middlewares/mock/mockSWIFTv2.go | 6 +++--- cns/middlewares/swiftV2.go | 2 +- cns/restserver/restserver.go | 4 ++-- cns/service/main.go | 2 +- 7 files changed, 15 insertions(+), 14 deletions(-) diff --git a/cns/api.go b/cns/api.go index 88bcc8045a..19f8a63da8 100644 --- a/cns/api.go +++ b/cns/api.go @@ -58,7 +58,7 @@ type SWIFTv2Middleware interface { SetRoutes(*PodIpInfo) error } -type IPConfigValidator func(context.Context, *IPConfigsRequest) (types.ResponseCode, string) +type IPConfigsRequestValidator func(context.Context, *IPConfigsRequest) (types.ResponseCode, string) // This is used for KubernetesCRD orchestrator Type where NC has multiple ips. // This struct captures the state for SecondaryIPs associated to a given NC diff --git a/cns/configuration/env.go b/cns/configuration/env.go index 3dbdc6d50a..eb80d9988e 100644 --- a/cns/configuration/env.go +++ b/cns/configuration/env.go @@ -11,11 +11,12 @@ const ( EnvNodeName = "NODENAME" // EnvNodeIP is the IP of the node running this CNS binary EnvNodeIP = "NODE_IP" - // LabelSwiftV2 is the Node label for Swift V2 - LabelSwiftV2 = "kubernetes.azure.com/podnetwork-multi-tenancy-enabled" - EnvPodV4CIDR = "POD_V4_CIDR" - EnvPodV6CIDR = "POD_V6_CIDR" - EnvServiceCIDR = "SERVICE_CIDR" + // LabelNodeSwiftV2 is the Node label for Swift V2 + LabelNodeSwiftV2 = "kubernetes.azure.com/podnetwork-multi-tenancy-enabled" + LabelPodSwiftV2 = "kubernetes.azure.com/pod-network" + EnvPodV4CIDR = "POD_V4_CIDR" + EnvPodV6CIDR = "POD_V6_CIDR" + EnvServiceCIDR = "SERVICE_CIDR" ) // ErrNodeNameUnset indicates the the $EnvNodeName variable is unset in the environment. diff --git a/cns/middlewares/mock/mockClient.go b/cns/middlewares/mock/mockClient.go index 5dac13bc31..79ef306197 100644 --- a/cns/middlewares/mock/mockClient.go +++ b/cns/middlewares/mock/mockClient.go @@ -26,7 +26,7 @@ type MockClient struct { func NewMockClient() *MockClient { testPod1 := v1.Pod{} testPod1.Labels = make(map[string]string) - testPod1.Labels[configuration.LabelSwiftV2] = "true" + testPod1.Labels[configuration.LabelNodeSwiftV2] = "true" testMTPNC1 := v1alpha1.MultitenantPodNetworkConfig{} testMTPNC1.Status.PrimaryIP = "192.168.0.1" diff --git a/cns/middlewares/mock/mockSWIFTv2.go b/cns/middlewares/mock/mockSWIFTv2.go index ba8c31c48e..dd0966c76e 100644 --- a/cns/middlewares/mock/mockSWIFTv2.go +++ b/cns/middlewares/mock/mockSWIFTv2.go @@ -26,7 +26,7 @@ type MockSWIFTv2Middleware struct { func NewMockSWIFTv2Middleware() *MockSWIFTv2Middleware { testPod1 := v1.Pod{} testPod1.Labels = make(map[string]string) - testPod1.Labels[configuration.LabelSwiftV2] = "true" + testPod1.Labels[configuration.LabelPodSwiftV2] = "true" testMTPNC1 := v1alpha1.MultitenantPodNetworkConfig{} testMTPNC1.Status.PrimaryIP = "192.168.0.1" @@ -55,8 +55,8 @@ func (m *MockSWIFTv2Middleware) ValidateIPConfigsRequest(_ context.Context, req errBuf := fmt.Sprintf("failed to get pod %v with error %v", podNamespacedName, err) return types.UnexpectedError, errBuf } - // check the pod labels for Swift V2, enrich the request with the multitenant flag. TBD on the label - if _, ok := pod.Labels[configuration.LabelSwiftV2]; ok { + // check the pod labels for Swift V2, enrich the request with the multitenant flag. + if _, ok := pod.Labels[configuration.LabelPodSwiftV2]; ok { req.SecondaryInterfacesExist = true } return types.Success, "" diff --git a/cns/middlewares/swiftV2.go b/cns/middlewares/swiftV2.go index affceef605..5855b07adb 100644 --- a/cns/middlewares/swiftV2.go +++ b/cns/middlewares/swiftV2.go @@ -47,7 +47,7 @@ func (m *SWIFTv2Middleware) ValidateIPConfigsRequest(ctx context.Context, req *c } // check the pod labels for Swift V2, set the request's SecondaryInterfaceSet flag to true. - if _, ok := pod.Labels[configuration.LabelSwiftV2]; ok { + if _, ok := pod.Labels[configuration.LabelPodSwiftV2]; ok { req.SecondaryInterfacesExist = true } return types.Success, "" diff --git a/cns/restserver/restserver.go b/cns/restserver/restserver.go index eb42b03932..08fff69664 100644 --- a/cns/restserver/restserver.go +++ b/cns/restserver/restserver.go @@ -72,7 +72,7 @@ type HTTPRestService struct { EndpointStateStore store.KeyValueStore cniConflistGenerator CNIConflistGenerator generateCNIConflistOnce sync.Once - ipConfigsRequestValidators []cns.IPConfigValidator + ipConfigsRequestValidators []cns.IPConfigsRequestValidator SWIFTv2Middleware cns.SWIFTv2Middleware } @@ -229,7 +229,7 @@ func (service *HTTPRestService) Init(config *common.ServiceConfig) error { } // Adding the default ipconfigs request validator - service.ipConfigsRequestValidators = []cns.IPConfigValidator{service.validateDefaultIPConfigsRequest} + service.ipConfigsRequestValidators = []cns.IPConfigsRequestValidator{service.validateDefaultIPConfigsRequest} // Add handlers. listener := service.Listener diff --git a/cns/service/main.go b/cns/service/main.go index 1b08f58a95..6bd6b44cea 100644 --- a/cns/service/main.go +++ b/cns/service/main.go @@ -1155,7 +1155,7 @@ func InitializeCRDState(ctx context.Context, httpRestService cns.HTTPService, cn } // check the Node labels for Swift V2 - if _, ok := node.Labels[configuration.LabelSwiftV2]; ok { + if _, ok := node.Labels[configuration.LabelNodeSwiftV2]; ok { cnsconfig.EnableSwiftV2 = true cnsconfig.WatchPods = true // TODO(rbtr): create the NodeInfo for Swift V2 From d294df35b0c598591f08f71ab46f0fe3ed7db43a Mon Sep 17 00:00:00 2001 From: Quang Nguyen Date: Tue, 3 Oct 2023 17:00:04 +0000 Subject: [PATCH 40/85] fix: fix swift v2 UT --- cns/middlewares/mock/mockClient.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cns/middlewares/mock/mockClient.go b/cns/middlewares/mock/mockClient.go index 79ef306197..6432d55852 100644 --- a/cns/middlewares/mock/mockClient.go +++ b/cns/middlewares/mock/mockClient.go @@ -26,7 +26,7 @@ type MockClient struct { func NewMockClient() *MockClient { testPod1 := v1.Pod{} testPod1.Labels = make(map[string]string) - testPod1.Labels[configuration.LabelNodeSwiftV2] = "true" + testPod1.Labels[configuration.LabelPodSwiftV2] = "true" testMTPNC1 := v1alpha1.MultitenantPodNetworkConfig{} testMTPNC1.Status.PrimaryIP = "192.168.0.1" From e4c60cf276713700047091705b331cb64569ffe9 Mon Sep 17 00:00:00 2001 From: Quang Nguyen Date: Tue, 3 Oct 2023 17:16:32 +0000 Subject: [PATCH 41/85] refractor: add v4/v6 distinction for service cidr --- cns/configuration/env.go | 28 ++++++++++++++++++++-------- cns/configuration/env_test.go | 20 +++++++++++++++----- cns/middlewares/swiftV2_test.go | 2 +- 3 files changed, 36 insertions(+), 14 deletions(-) diff --git a/cns/configuration/env.go b/cns/configuration/env.go index eb80d9988e..85d00b4c5e 100644 --- a/cns/configuration/env.go +++ b/cns/configuration/env.go @@ -16,7 +16,8 @@ const ( LabelPodSwiftV2 = "kubernetes.azure.com/pod-network" EnvPodV4CIDR = "POD_V4_CIDR" EnvPodV6CIDR = "POD_V6_CIDR" - EnvServiceCIDR = "SERVICE_CIDR" + EnvServiceV4CIDR = "SERVICE_V4_CIDR" + EnvServiceV6CIDR = "SERVICE_V6_CIDR" ) // ErrNodeNameUnset indicates the the $EnvNodeName variable is unset in the environment. @@ -28,8 +29,11 @@ var ErrPodV4CIDRUnset = errors.Errorf("must declare %s environment variable", En // ErrPodV6CIDRUnset indicates the the $EnvPodV6CIDR variable is unset in the environment. var ErrPodV6CIDRUnset = errors.Errorf("must declare %s environment variable", EnvPodV6CIDR) -// ErrServiceCIDRUnset indicates the the $EnvServiceCIDR variable is unset in the environment. -var ErrServiceCIDRUnset = errors.Errorf("must declare %s environment variable", EnvServiceCIDR) +// ErrServiceV4CIDRUnset indicates the the $EnvServiceV4CIDR variable is unset in the environment. +var ErrServiceV4CIDRUnset = errors.Errorf("must declare %s environment variable", EnvServiceV4CIDR) + +// ErrServiceV6CIDRUnset indicates the the $EnvServiceV6CIDR variable is unset in the environment. +var ErrServiceV6CIDRUnset = errors.Errorf("must declare %s environment variable", EnvServiceV6CIDR) // NodeName checks the environment variables for the NODENAME and returns it or an error if unset. func NodeName() (string, error) { @@ -61,10 +65,18 @@ func PodV6CIDR() (string, error) { return podCIDRv6, nil } -func ServiceCIDR() (string, error) { - serviceCIDR := os.Getenv(EnvServiceCIDR) - if serviceCIDR == "" { - return "", ErrServiceCIDRUnset +func ServiceV4CIDR() (string, error) { + serviceV4CIDR := os.Getenv(EnvServiceV4CIDR) + if serviceV4CIDR == "" { + return "", ErrServiceV4CIDRUnset + } + return serviceV4CIDR, nil +} + +func ServiceV6CIDR() (string, error) { + serviceV6CIDR := os.Getenv(EnvServiceV6CIDR) + if serviceV6CIDR == "" { + return "", ErrServiceV6CIDRUnset } - return serviceCIDR, nil + return serviceV6CIDR, nil } diff --git a/cns/configuration/env_test.go b/cns/configuration/env_test.go index 5432c467f9..afebcf8ab4 100644 --- a/cns/configuration/env_test.go +++ b/cns/configuration/env_test.go @@ -38,12 +38,22 @@ func TestPodV6CIDR(t *testing.T) { assert.Equal(t, "test", cidr) } -func TestServiceCIDR(t *testing.T) { - _, err := ServiceCIDR() +func TestServiceV4CIDR(t *testing.T) { + _, err := ServiceV4CIDR() require.Error(t, err) - require.ErrorIs(t, err, ErrServiceCIDRUnset) - os.Setenv(EnvServiceCIDR, "test") - cidr, err := ServiceCIDR() + require.ErrorIs(t, err, ErrServiceV4CIDRUnset) + os.Setenv(EnvServiceV4CIDR, "test") + cidr, err := ServiceV4CIDR() + assert.NoError(t, err) + assert.Equal(t, "test", cidr) +} + +func TestServiceV6CIDR(t *testing.T) { + _, err := ServiceV6CIDR() + require.Error(t, err) + require.ErrorIs(t, err, ErrServiceV6CIDRUnset) + os.Setenv(EnvServiceV6CIDR, "test") + cidr, err := ServiceV6CIDR() assert.NoError(t, err) assert.Equal(t, "test", cidr) } diff --git a/cns/middlewares/swiftV2_test.go b/cns/middlewares/swiftV2_test.go index 0f4284d224..e18ad0db65 100644 --- a/cns/middlewares/swiftV2_test.go +++ b/cns/middlewares/swiftV2_test.go @@ -65,7 +65,7 @@ func TestValidateMultitenantIPConfigsRequestFailure(t *testing.T) { func TestGetSWIFTv2IPConfigSuccess(t *testing.T) { os.Setenv(configuration.EnvPodV4CIDR, "10.0.1.10/24") - os.Setenv(configuration.EnvServiceCIDR, "10.0.2.10/24") + os.Setenv(configuration.EnvServiceV4CIDR, "10.0.2.10/24") middleware := SWIFTv2Middleware{Cli: mock.NewMockClient()} From 511309427a044d5d88d5f00707316b3cdb26e58c Mon Sep 17 00:00:00 2001 From: Quang Nguyen Date: Tue, 29 Aug 2023 16:10:47 +0000 Subject: [PATCH 42/85] rebase --- .../{buildinfo/buildinfo.go => buildversion/buildversion.go} | 4 ++-- azure-ipam/ipam.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) rename azure-ipam/internal/{buildinfo/buildinfo.go => buildversion/buildversion.go} (74%) diff --git a/azure-ipam/internal/buildinfo/buildinfo.go b/azure-ipam/internal/buildversion/buildversion.go similarity index 74% rename from azure-ipam/internal/buildinfo/buildinfo.go rename to azure-ipam/internal/buildversion/buildversion.go index e26b8d2f8e..e25cccd172 100644 --- a/azure-ipam/internal/buildinfo/buildinfo.go +++ b/azure-ipam/internal/buildversion/buildversion.go @@ -1,6 +1,6 @@ -package buildinfo +package buildversion // this will be populate by the Go compiler via // the -ldflags, which insert dynamic information // into the binary at build time -var Version string +var BuildVersion string diff --git a/azure-ipam/ipam.go b/azure-ipam/ipam.go index ed0101bbb3..b45114f38c 100644 --- a/azure-ipam/ipam.go +++ b/azure-ipam/ipam.go @@ -7,7 +7,7 @@ import ( "io" "net" - "github.com/Azure/azure-container-networking/azure-ipam/internal/buildinfo" + "github.com/Azure/azure-container-networking/azure-ipam/internal/buildversion" "github.com/Azure/azure-container-networking/azure-ipam/ipconfig" "github.com/Azure/azure-container-networking/cns" cnscli "github.com/Azure/azure-container-networking/cns/client" @@ -43,7 +43,7 @@ type cnsClient interface { func NewPlugin(logger *zap.Logger, c cnsClient, out io.Writer) (*IPAMPlugin, error) { plugin := &IPAMPlugin{ Name: pluginName, - Version: buildinfo.Version, + Version: buildversion.BuildVersion, logger: logger, out: out, cnsClient: c, From aee4f2eac9a38c1eae2afa4f53e9cc30ae17fbcf Mon Sep 17 00:00:00 2001 From: Quang Nguyen Date: Mon, 31 Jul 2023 22:09:59 +0000 Subject: [PATCH 43/85] revert file and var naming, add correct path to makefile --- .../{buildversion/buildversion.go => buildinfo/buildinfo.go} | 4 ++-- azure-ipam/ipam.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) rename azure-ipam/internal/{buildversion/buildversion.go => buildinfo/buildinfo.go} (74%) diff --git a/azure-ipam/internal/buildversion/buildversion.go b/azure-ipam/internal/buildinfo/buildinfo.go similarity index 74% rename from azure-ipam/internal/buildversion/buildversion.go rename to azure-ipam/internal/buildinfo/buildinfo.go index e25cccd172..e26b8d2f8e 100644 --- a/azure-ipam/internal/buildversion/buildversion.go +++ b/azure-ipam/internal/buildinfo/buildinfo.go @@ -1,6 +1,6 @@ -package buildversion +package buildinfo // this will be populate by the Go compiler via // the -ldflags, which insert dynamic information // into the binary at build time -var BuildVersion string +var Version string diff --git a/azure-ipam/ipam.go b/azure-ipam/ipam.go index b45114f38c..ed0101bbb3 100644 --- a/azure-ipam/ipam.go +++ b/azure-ipam/ipam.go @@ -7,7 +7,7 @@ import ( "io" "net" - "github.com/Azure/azure-container-networking/azure-ipam/internal/buildversion" + "github.com/Azure/azure-container-networking/azure-ipam/internal/buildinfo" "github.com/Azure/azure-container-networking/azure-ipam/ipconfig" "github.com/Azure/azure-container-networking/cns" cnscli "github.com/Azure/azure-container-networking/cns/client" @@ -43,7 +43,7 @@ type cnsClient interface { func NewPlugin(logger *zap.Logger, c cnsClient, out io.Writer) (*IPAMPlugin, error) { plugin := &IPAMPlugin{ Name: pluginName, - Version: buildversion.BuildVersion, + Version: buildinfo.Version, logger: logger, out: out, cnsClient: c, From 68e65e70bcc3ad914865acaebfa20db01060c2bd Mon Sep 17 00:00:00 2001 From: Quang Nguyen Date: Tue, 29 Aug 2023 16:10:47 +0000 Subject: [PATCH 44/85] rebase --- .../{buildinfo/buildinfo.go => buildversion/buildversion.go} | 4 ++-- azure-ipam/ipam.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) rename azure-ipam/internal/{buildinfo/buildinfo.go => buildversion/buildversion.go} (74%) diff --git a/azure-ipam/internal/buildinfo/buildinfo.go b/azure-ipam/internal/buildversion/buildversion.go similarity index 74% rename from azure-ipam/internal/buildinfo/buildinfo.go rename to azure-ipam/internal/buildversion/buildversion.go index e26b8d2f8e..e25cccd172 100644 --- a/azure-ipam/internal/buildinfo/buildinfo.go +++ b/azure-ipam/internal/buildversion/buildversion.go @@ -1,6 +1,6 @@ -package buildinfo +package buildversion // this will be populate by the Go compiler via // the -ldflags, which insert dynamic information // into the binary at build time -var Version string +var BuildVersion string diff --git a/azure-ipam/ipam.go b/azure-ipam/ipam.go index ed0101bbb3..b45114f38c 100644 --- a/azure-ipam/ipam.go +++ b/azure-ipam/ipam.go @@ -7,7 +7,7 @@ import ( "io" "net" - "github.com/Azure/azure-container-networking/azure-ipam/internal/buildinfo" + "github.com/Azure/azure-container-networking/azure-ipam/internal/buildversion" "github.com/Azure/azure-container-networking/azure-ipam/ipconfig" "github.com/Azure/azure-container-networking/cns" cnscli "github.com/Azure/azure-container-networking/cns/client" @@ -43,7 +43,7 @@ type cnsClient interface { func NewPlugin(logger *zap.Logger, c cnsClient, out io.Writer) (*IPAMPlugin, error) { plugin := &IPAMPlugin{ Name: pluginName, - Version: buildinfo.Version, + Version: buildversion.BuildVersion, logger: logger, out: out, cnsClient: c, From fdebcf4e46d20f5ebd57fcad868b115c7d53a9fd Mon Sep 17 00:00:00 2001 From: Quang Nguyen Date: Mon, 31 Jul 2023 22:09:59 +0000 Subject: [PATCH 45/85] revert file and var naming, add correct path to makefile --- .../{buildversion/buildversion.go => buildinfo/buildinfo.go} | 4 ++-- azure-ipam/ipam.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) rename azure-ipam/internal/{buildversion/buildversion.go => buildinfo/buildinfo.go} (74%) diff --git a/azure-ipam/internal/buildversion/buildversion.go b/azure-ipam/internal/buildinfo/buildinfo.go similarity index 74% rename from azure-ipam/internal/buildversion/buildversion.go rename to azure-ipam/internal/buildinfo/buildinfo.go index e25cccd172..e26b8d2f8e 100644 --- a/azure-ipam/internal/buildversion/buildversion.go +++ b/azure-ipam/internal/buildinfo/buildinfo.go @@ -1,6 +1,6 @@ -package buildversion +package buildinfo // this will be populate by the Go compiler via // the -ldflags, which insert dynamic information // into the binary at build time -var BuildVersion string +var Version string diff --git a/azure-ipam/ipam.go b/azure-ipam/ipam.go index b45114f38c..ed0101bbb3 100644 --- a/azure-ipam/ipam.go +++ b/azure-ipam/ipam.go @@ -7,7 +7,7 @@ import ( "io" "net" - "github.com/Azure/azure-container-networking/azure-ipam/internal/buildversion" + "github.com/Azure/azure-container-networking/azure-ipam/internal/buildinfo" "github.com/Azure/azure-container-networking/azure-ipam/ipconfig" "github.com/Azure/azure-container-networking/cns" cnscli "github.com/Azure/azure-container-networking/cns/client" @@ -43,7 +43,7 @@ type cnsClient interface { func NewPlugin(logger *zap.Logger, c cnsClient, out io.Writer) (*IPAMPlugin, error) { plugin := &IPAMPlugin{ Name: pluginName, - Version: buildversion.BuildVersion, + Version: buildinfo.Version, logger: logger, out: out, cnsClient: c, From f4802ebb95fe0e5dabdfe284e9e9a9e8f3de5bdb Mon Sep 17 00:00:00 2001 From: Quang Nguyen Date: Wed, 9 Aug 2023 20:28:51 +0000 Subject: [PATCH 46/85] change podipinfo + linter issue --- cns/middlewares/multitenant_validator.go | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 cns/middlewares/multitenant_validator.go diff --git a/cns/middlewares/multitenant_validator.go b/cns/middlewares/multitenant_validator.go new file mode 100644 index 0000000000..ae31adaf05 --- /dev/null +++ b/cns/middlewares/multitenant_validator.go @@ -0,0 +1,23 @@ +package middlewares + +import ( + "github.com/Azure/azure-container-networking/cns" + "github.com/Azure/azure-container-networking/cns/types" +) + +type MultitenantValidator struct { + // TODO: implement + // need cached scoped client for pods +} + +func NewMultitenantValidator() *MultitenantValidator { + return &MultitenantValidator{} +} + +// validateMultitenantIPConfigsRequest validate whether the request is for a multitenant pod +// nolint +func (v *MultitenantValidator) validateMultitenantIPConfigsRequest(ipConfigsRequest *cns.IPConfigsRequest) (respCode types.ResponseCode, message string) { + // TODO: if pod is multitenant, enrich the request with the multitenant flag + ipConfigsRequest.Multitenant = true + return types.Success, "" +} From ece68a692735aa0acc59caca7e8b79dcf195c7cd Mon Sep 17 00:00:00 2001 From: Quang Nguyen Date: Tue, 15 Aug 2023 20:08:57 +0000 Subject: [PATCH 47/85] update IPAM branching --- cns/middlewares/multitenant.go | 50 ++++++++++++++++++++++++ cns/middlewares/multitenant_validator.go | 23 ----------- cns/restserver/ipam.go | 28 +++++++++++++ cns/restserver/restserver.go | 13 ++++++ cns/service/main.go | 5 +++ 5 files changed, 96 insertions(+), 23 deletions(-) create mode 100644 cns/middlewares/multitenant.go delete mode 100644 cns/middlewares/multitenant_validator.go diff --git a/cns/middlewares/multitenant.go b/cns/middlewares/multitenant.go new file mode 100644 index 0000000000..e0ddf4d1f6 --- /dev/null +++ b/cns/middlewares/multitenant.go @@ -0,0 +1,50 @@ +package middlewares + +import ( + "github.com/Azure/azure-container-networking/cns" + "github.com/Azure/azure-container-networking/cns/types" +) + +type IPConfigValidator func(ipConfigsRequest *cns.IPConfigsRequest) (types.ResponseCode, string) + +// Middleware interface for testing later on +type Middleware interface { + Validator() IPConfigValidator + GetMultitenantIPConfig(podInfo cns.PodInfo) (*cns.PodIpInfo, error) +} + +type MultitenantMiddleware struct { + // TODO: implement + // need cached scoped client for pods + // need client for MTPNC CRD for x-ref pods +} + +func NewMultitenantMiddleware() *MultitenantMiddleware { + return &MultitenantMiddleware{} +} + +// Return the validator function for the middleware +func (m *MultitenantMiddleware) Validator() IPConfigValidator { + return m.validateMultitenantIPConfigsRequest +} + +// validateMultitenantIPConfigsRequest validate whether the request is for a multitenant pod +// nolint +func (m *MultitenantMiddleware) validateMultitenantIPConfigsRequest(ipConfigsRequest *cns.IPConfigsRequest) (respCode types.ResponseCode, message string) { + /** + TODO: + - Check if pod is multitenant, enrich the request with the multitenant flag + **/ + ipConfigsRequest.Multitenant = true + return types.Success, "" +} + +// nolint +// GetMultitenantIPConfig returns the IP config for a multitenant pod from the MTPNC CRD +func (m *MultitenantMiddleware) GetMultitenantIPConfig(podInfo cns.PodInfo) (*cns.PodIpInfo, error) { + /** + TODO: + - Check if the MTPNC CRD exists for the pod, if not, return error + **/ + return nil, nil +} diff --git a/cns/middlewares/multitenant_validator.go b/cns/middlewares/multitenant_validator.go deleted file mode 100644 index ae31adaf05..0000000000 --- a/cns/middlewares/multitenant_validator.go +++ /dev/null @@ -1,23 +0,0 @@ -package middlewares - -import ( - "github.com/Azure/azure-container-networking/cns" - "github.com/Azure/azure-container-networking/cns/types" -) - -type MultitenantValidator struct { - // TODO: implement - // need cached scoped client for pods -} - -func NewMultitenantValidator() *MultitenantValidator { - return &MultitenantValidator{} -} - -// validateMultitenantIPConfigsRequest validate whether the request is for a multitenant pod -// nolint -func (v *MultitenantValidator) validateMultitenantIPConfigsRequest(ipConfigsRequest *cns.IPConfigsRequest) (respCode types.ResponseCode, message string) { - // TODO: if pod is multitenant, enrich the request with the multitenant flag - ipConfigsRequest.Multitenant = true - return types.Success, "" -} diff --git a/cns/restserver/ipam.go b/cns/restserver/ipam.go index 2e51ef6bde..675bfe7b73 100644 --- a/cns/restserver/ipam.go +++ b/cns/restserver/ipam.go @@ -37,6 +37,34 @@ func (service *HTTPRestService) requestIPConfigHandlerHelper(ctx context.Context }, errors.New("failed to validate ip config request") } + var MTNpodIPInfo *cns.PodIpInfo + + // Request is for multitenant pod + if ipconfigsRequest.Multitenant { + // Multitenant middleware not set + if service.MultitenantMiddleware == nil { + return &cns.IPConfigsResponse{ + Response: cns.Response{ + ReturnCode: types.FailedToAllocateIPConfig, + Message: fmt.Sprintf("AllocateIPConfig failed: %v, IP config request is %v", errors.New("request is for multitenant pod but multitenant middleware is nil"), ipconfigsRequest), + }, + PodIPInfo: []cns.PodIpInfo{}, + }, errors.New("request is for multitenant pod but multitenant middleware is nil") + } + // If pod is multitenant and we failed to grab its MTPNC IP config, return error immediately + podIPInfo, err := service.MultitenantMiddleware.GetMultitenantIPConfig(podInfo) + if err != nil { + return &cns.IPConfigsResponse{ + Response: cns.Response{ + ReturnCode: types.FailedToAllocateIPConfig, + Message: fmt.Sprintf("AllocateIPConfig failed: %v, IP config request is %v", err, ipconfigsRequest), + }, + PodIPInfo: []cns.PodIpInfo{}, + }, errors.Wrapf(err, "failed to get multitenant IP config %v", ipconfigsRequest) + } + MTNpodIPInfo = podIPInfo + } + // record a pod requesting an IP service.podsPendingIPAssignment.Push(podInfo.Key()) diff --git a/cns/restserver/restserver.go b/cns/restserver/restserver.go index 08fff69664..66d7dcb8cb 100644 --- a/cns/restserver/restserver.go +++ b/cns/restserver/restserver.go @@ -13,6 +13,7 @@ import ( "github.com/Azure/azure-container-networking/cns/dockerclient" "github.com/Azure/azure-container-networking/cns/ipamclient" "github.com/Azure/azure-container-networking/cns/logger" + "github.com/Azure/azure-container-networking/cns/middlewares" "github.com/Azure/azure-container-networking/cns/networkcontainers" "github.com/Azure/azure-container-networking/cns/routes" "github.com/Azure/azure-container-networking/cns/types" @@ -228,8 +229,13 @@ func (service *HTTPRestService) Init(config *common.ServiceConfig) error { return err } +<<<<<<< HEAD // Adding the default ipconfigs request validator service.ipConfigsRequestValidators = []cns.IPConfigsRequestValidator{service.validateDefaultIPConfigsRequest} +======= + // Adding ipConfigsValidators + service.ipConfigsValidators = []middlewares.IPConfigValidator{service.validateDefaultIPConfigsRequest} +>>>>>>> ab16c636 (update IPAM branching) // Add handlers. listener := service.Listener @@ -353,8 +359,15 @@ func (service *HTTPRestService) MustGenerateCNIConflistOnce() { }) } +<<<<<<< HEAD func (service *HTTPRestService) AttachSWIFTv2Middleware(middleware cns.SWIFTv2Middleware) { service.SWIFTv2Middleware = middleware // adding the SWIFT v2 ipconfigs request validator service.ipConfigsRequestValidators = append(service.ipConfigsRequestValidators, middleware.ValidateIPConfigsRequest) +======= +func (service *HTTPRestService) AttachMultitenantMiddleware(middleware middlewares.Middleware) { + service.MultitenantMiddleware = middleware + // add multitenant ipconfig validator function + service.ipConfigsValidators = append(service.ipConfigsValidators, middleware.Validator()) +>>>>>>> ab16c636 (update IPAM branching) } diff --git a/cns/service/main.go b/cns/service/main.go index 6bd6b44cea..dedbd443ee 100644 --- a/cns/service/main.go +++ b/cns/service/main.go @@ -804,6 +804,11 @@ func main() { if cnsconfig.EnablePprof { httpRestService.RegisterPProfEndpoints() } + // if SWIFT v2 is enabled on CNS, attach multitenant middleware to rest service + if cnsconfig.EnableSwiftV2 { + multitenantMiddleware := middlewares.NewMultitenantMiddleware() + httpRestService.AttachMultitenantMiddleware(multitenantMiddleware) + } err = httpRestService.Start(&config) if err != nil { From ddf472a82c21b771ad0fd5c0e503f41d52b9e460 Mon Sep 17 00:00:00 2001 From: Quang Nguyen Date: Fri, 18 Aug 2023 18:27:16 +0000 Subject: [PATCH 48/85] pod client placeholder --- cns/api.go | 13 +++++++++++++ cns/middlewares/multitenant.go | 18 +++++++----------- cns/restserver/restserver.go | 12 ++++-------- cns/service/main.go | 6 ------ 4 files changed, 24 insertions(+), 25 deletions(-) diff --git a/cns/api.go b/cns/api.go index 19f8a63da8..a98084b858 100644 --- a/cns/api.go +++ b/cns/api.go @@ -48,6 +48,7 @@ type HTTPService interface { GetPendingReleaseIPConfigs() []IPConfigurationStatus GetPodIPConfigState() map[string]IPConfigurationStatus MarkIPAsPendingRelease(numberToMark int) (map[string]IPConfigurationStatus, error) +<<<<<<< HEAD AttachSWIFTv2Middleware(middleware SWIFTv2Middleware) } @@ -59,6 +60,18 @@ type SWIFTv2Middleware interface { } type IPConfigsRequestValidator func(context.Context, *IPConfigsRequest) (types.ResponseCode, string) +======= + AttachMultitenantMiddleware(middleware MultitenantMiddleware) +} + +// Middleware interface for testing later on +type MultitenantMiddleware interface { + Validator() IPConfigValidator + GetMultitenantIPConfig(podInfo PodInfo) (*PodIpInfo, error) +} + +type IPConfigValidator func(ipConfigsRequest *IPConfigsRequest) (types.ResponseCode, string) +>>>>>>> 9e461501 (pod client placeholder) // This is used for KubernetesCRD orchestrator Type where NC has multiple ips. // This struct captures the state for SecondaryIPs associated to a given NC diff --git a/cns/middlewares/multitenant.go b/cns/middlewares/multitenant.go index e0ddf4d1f6..94ad1956c0 100644 --- a/cns/middlewares/multitenant.go +++ b/cns/middlewares/multitenant.go @@ -3,28 +3,24 @@ package middlewares import ( "github.com/Azure/azure-container-networking/cns" "github.com/Azure/azure-container-networking/cns/types" + "sigs.k8s.io/controller-runtime/pkg/client" ) -type IPConfigValidator func(ipConfigsRequest *cns.IPConfigsRequest) (types.ResponseCode, string) - -// Middleware interface for testing later on -type Middleware interface { - Validator() IPConfigValidator - GetMultitenantIPConfig(podInfo cns.PodInfo) (*cns.PodIpInfo, error) -} - type MultitenantMiddleware struct { // TODO: implement // need cached scoped client for pods // need client for MTPNC CRD for x-ref pods + cli client.Client } -func NewMultitenantMiddleware() *MultitenantMiddleware { - return &MultitenantMiddleware{} +func NewMultitenantMiddleware(cli client.Client) *MultitenantMiddleware { + return &MultitenantMiddleware{ + cli: cli, + } } // Return the validator function for the middleware -func (m *MultitenantMiddleware) Validator() IPConfigValidator { +func (m *MultitenantMiddleware) Validator() cns.IPConfigValidator { return m.validateMultitenantIPConfigsRequest } diff --git a/cns/restserver/restserver.go b/cns/restserver/restserver.go index 66d7dcb8cb..689f4d2d95 100644 --- a/cns/restserver/restserver.go +++ b/cns/restserver/restserver.go @@ -13,7 +13,6 @@ import ( "github.com/Azure/azure-container-networking/cns/dockerclient" "github.com/Azure/azure-container-networking/cns/ipamclient" "github.com/Azure/azure-container-networking/cns/logger" - "github.com/Azure/azure-container-networking/cns/middlewares" "github.com/Azure/azure-container-networking/cns/networkcontainers" "github.com/Azure/azure-container-networking/cns/routes" "github.com/Azure/azure-container-networking/cns/types" @@ -234,8 +233,12 @@ func (service *HTTPRestService) Init(config *common.ServiceConfig) error { service.ipConfigsRequestValidators = []cns.IPConfigsRequestValidator{service.validateDefaultIPConfigsRequest} ======= // Adding ipConfigsValidators +<<<<<<< HEAD service.ipConfigsValidators = []middlewares.IPConfigValidator{service.validateDefaultIPConfigsRequest} >>>>>>> ab16c636 (update IPAM branching) +======= + service.ipConfigsValidators = []cns.IPConfigValidator{service.validateDefaultIPConfigsRequest} +>>>>>>> 9e461501 (pod client placeholder) // Add handlers. listener := service.Listener @@ -359,15 +362,8 @@ func (service *HTTPRestService) MustGenerateCNIConflistOnce() { }) } -<<<<<<< HEAD func (service *HTTPRestService) AttachSWIFTv2Middleware(middleware cns.SWIFTv2Middleware) { service.SWIFTv2Middleware = middleware // adding the SWIFT v2 ipconfigs request validator service.ipConfigsRequestValidators = append(service.ipConfigsRequestValidators, middleware.ValidateIPConfigsRequest) -======= -func (service *HTTPRestService) AttachMultitenantMiddleware(middleware middlewares.Middleware) { - service.MultitenantMiddleware = middleware - // add multitenant ipconfig validator function - service.ipConfigsValidators = append(service.ipConfigsValidators, middleware.Validator()) ->>>>>>> ab16c636 (update IPAM branching) } diff --git a/cns/service/main.go b/cns/service/main.go index dedbd443ee..cc5ea6f2c0 100644 --- a/cns/service/main.go +++ b/cns/service/main.go @@ -37,7 +37,6 @@ import ( nncctrl "github.com/Azure/azure-container-networking/cns/kubecontroller/nodenetworkconfig" podctrl "github.com/Azure/azure-container-networking/cns/kubecontroller/pod" "github.com/Azure/azure-container-networking/cns/logger" - "github.com/Azure/azure-container-networking/cns/middlewares" "github.com/Azure/azure-container-networking/cns/multitenantcontroller" "github.com/Azure/azure-container-networking/cns/multitenantcontroller/multitenantoperator" "github.com/Azure/azure-container-networking/cns/restserver" @@ -804,11 +803,6 @@ func main() { if cnsconfig.EnablePprof { httpRestService.RegisterPProfEndpoints() } - // if SWIFT v2 is enabled on CNS, attach multitenant middleware to rest service - if cnsconfig.EnableSwiftV2 { - multitenantMiddleware := middlewares.NewMultitenantMiddleware() - httpRestService.AttachMultitenantMiddleware(multitenantMiddleware) - } err = httpRestService.Start(&config) if err != nil { From 691f73390472b058d2e1babc061c3c0dc362d095 Mon Sep 17 00:00:00 2001 From: Quang Nguyen Date: Mon, 21 Aug 2023 20:21:10 +0000 Subject: [PATCH 49/85] getting pod info in validator --- cns/middlewares/multitenant.go | 33 ++++++++++++++++++++++++++------- cns/service/main.go | 19 +++++++++++++++++++ 2 files changed, 45 insertions(+), 7 deletions(-) diff --git a/cns/middlewares/multitenant.go b/cns/middlewares/multitenant.go index 94ad1956c0..fff83f75e7 100644 --- a/cns/middlewares/multitenant.go +++ b/cns/middlewares/multitenant.go @@ -1,8 +1,14 @@ package middlewares import ( + "context" + "fmt" + "github.com/Azure/azure-container-networking/cns" + "github.com/Azure/azure-container-networking/cns/configuration" "github.com/Azure/azure-container-networking/cns/types" + v1 "k8s.io/api/core/v1" + k8types "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -24,14 +30,27 @@ func (m *MultitenantMiddleware) Validator() cns.IPConfigValidator { return m.validateMultitenantIPConfigsRequest } -// validateMultitenantIPConfigsRequest validate whether the request is for a multitenant pod +// validateMultitenantIPConfigsRequest validates if pod is multitenant // nolint -func (m *MultitenantMiddleware) validateMultitenantIPConfigsRequest(ipConfigsRequest *cns.IPConfigsRequest) (respCode types.ResponseCode, message string) { - /** - TODO: - - Check if pod is multitenant, enrich the request with the multitenant flag - **/ - ipConfigsRequest.Multitenant = true +func (m *MultitenantMiddleware) validateMultitenantIPConfigsRequest(req *cns.IPConfigsRequest) (respCode types.ResponseCode, message string) { + // Retrieve the pod from the cluster + podInfo, err := cns.UnmarshalPodInfo(req.OrchestratorContext) + if err != nil { + errBuf := fmt.Sprintf("unmarshalling pod info from ipconfigs request %v failed with error %v", req, err) + return types.UnexpectedError, errBuf + } + podNamespacedName := k8types.NamespacedName{Namespace: podInfo.Namespace(), Name: podInfo.Name()} + pod := v1.Pod{} + err = m.cli.Get(context.TODO(), podNamespacedName, &pod) + if err != nil { + errBuf := fmt.Sprintf("failed to get pod %v with error %v", podNamespacedName, err) + return types.UnexpectedError, errBuf + } + + // check the pod labels for Swift V2, enrich the request with the multitenant flag. TBD on the label + if _, ok := pod.Labels[configuration.LabelSwiftV2]; ok { + req.Multitenant = true + } return types.Success, "" } diff --git a/cns/service/main.go b/cns/service/main.go index cc5ea6f2c0..da837b825b 100644 --- a/cns/service/main.go +++ b/cns/service/main.go @@ -59,7 +59,12 @@ import ( "github.com/avast/retry-go/v3" "github.com/pkg/errors" "go.uber.org/zap" +<<<<<<< HEAD corev1 "k8s.io/api/core/v1" +======= + corev1 "k8s.io/api/apps/v1" + v1 "k8s.io/api/core/v1" +>>>>>>> f2359ac8 (getting pod info in validator) apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" @@ -1252,7 +1257,21 @@ func InitializeCRDState(ctx context.Context, httpRestService cns.HTTPService, cn "kube-system": {FieldSelector: fields.SelectorFromSet(fields.Set{"metadata.name": nodeName})}, }, }, + &v1.Pod{}: { + Field: fields.SelectorFromSet(fields.Set{"spec.nodeName": nodeName}), + }, }, + }) + + crdSchemes := kuberuntime.NewScheme() + if err = v1alpha.AddToScheme(crdSchemes); err != nil { + return errors.Wrap(err, "failed to add nodenetworkconfig/v1alpha to scheme") + } + if err = v1alpha1.AddToScheme(crdSchemes); err != nil { + return errors.Wrap(err, "failed to add clustersubnetstate/v1alpha1 to scheme") + } + if err = corev1.AddToScheme(crdSchemes); err != nil { + return errors.Wrap(err, "failed to add core/v1 to scheme") } if cnsconfig.WatchPods { From e11247b8f995208123b77471aeac1bf6a1ce817e Mon Sep 17 00:00:00 2001 From: Quang Nguyen Date: Mon, 21 Aug 2023 20:38:18 +0000 Subject: [PATCH 50/85] linter issue --- cns/service/main.go | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/cns/service/main.go b/cns/service/main.go index da837b825b..0734773cef 100644 --- a/cns/service/main.go +++ b/cns/service/main.go @@ -59,12 +59,7 @@ import ( "github.com/avast/retry-go/v3" "github.com/pkg/errors" "go.uber.org/zap" -<<<<<<< HEAD corev1 "k8s.io/api/core/v1" -======= - corev1 "k8s.io/api/apps/v1" - v1 "k8s.io/api/core/v1" ->>>>>>> f2359ac8 (getting pod info in validator) apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" @@ -1257,17 +1252,17 @@ func InitializeCRDState(ctx context.Context, httpRestService cns.HTTPService, cn "kube-system": {FieldSelector: fields.SelectorFromSet(fields.Set{"metadata.name": nodeName})}, }, }, - &v1.Pod{}: { + &corev1.Pod{}: { // nolint: typecheck Field: fields.SelectorFromSet(fields.Set{"spec.nodeName": nodeName}), }, }, - }) + } crdSchemes := kuberuntime.NewScheme() if err = v1alpha.AddToScheme(crdSchemes); err != nil { return errors.Wrap(err, "failed to add nodenetworkconfig/v1alpha to scheme") } - if err = v1alpha1.AddToScheme(crdSchemes); err != nil { + if err = v1alpha.AddToScheme(crdSchemes); err != nil { return errors.Wrap(err, "failed to add clustersubnetstate/v1alpha1 to scheme") } if err = corev1.AddToScheme(crdSchemes); err != nil { From b6d6c26b25a9a9c19194c613870aef3cb29d41f9 Mon Sep 17 00:00:00 2001 From: Quang Nguyen Date: Tue, 29 Aug 2023 16:10:47 +0000 Subject: [PATCH 51/85] rebase --- .../{buildinfo/buildinfo.go => buildversion/buildversion.go} | 4 ++-- azure-ipam/ipam.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) rename azure-ipam/internal/{buildinfo/buildinfo.go => buildversion/buildversion.go} (74%) diff --git a/azure-ipam/internal/buildinfo/buildinfo.go b/azure-ipam/internal/buildversion/buildversion.go similarity index 74% rename from azure-ipam/internal/buildinfo/buildinfo.go rename to azure-ipam/internal/buildversion/buildversion.go index e26b8d2f8e..e25cccd172 100644 --- a/azure-ipam/internal/buildinfo/buildinfo.go +++ b/azure-ipam/internal/buildversion/buildversion.go @@ -1,6 +1,6 @@ -package buildinfo +package buildversion // this will be populate by the Go compiler via // the -ldflags, which insert dynamic information // into the binary at build time -var Version string +var BuildVersion string diff --git a/azure-ipam/ipam.go b/azure-ipam/ipam.go index ed0101bbb3..b45114f38c 100644 --- a/azure-ipam/ipam.go +++ b/azure-ipam/ipam.go @@ -7,7 +7,7 @@ import ( "io" "net" - "github.com/Azure/azure-container-networking/azure-ipam/internal/buildinfo" + "github.com/Azure/azure-container-networking/azure-ipam/internal/buildversion" "github.com/Azure/azure-container-networking/azure-ipam/ipconfig" "github.com/Azure/azure-container-networking/cns" cnscli "github.com/Azure/azure-container-networking/cns/client" @@ -43,7 +43,7 @@ type cnsClient interface { func NewPlugin(logger *zap.Logger, c cnsClient, out io.Writer) (*IPAMPlugin, error) { plugin := &IPAMPlugin{ Name: pluginName, - Version: buildinfo.Version, + Version: buildversion.BuildVersion, logger: logger, out: out, cnsClient: c, From 48a911d3843d1ac236c488722deb930845380cf6 Mon Sep 17 00:00:00 2001 From: Quang Nguyen Date: Mon, 31 Jul 2023 22:09:59 +0000 Subject: [PATCH 52/85] revert file and var naming, add correct path to makefile --- .../{buildversion/buildversion.go => buildinfo/buildinfo.go} | 4 ++-- azure-ipam/ipam.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) rename azure-ipam/internal/{buildversion/buildversion.go => buildinfo/buildinfo.go} (74%) diff --git a/azure-ipam/internal/buildversion/buildversion.go b/azure-ipam/internal/buildinfo/buildinfo.go similarity index 74% rename from azure-ipam/internal/buildversion/buildversion.go rename to azure-ipam/internal/buildinfo/buildinfo.go index e25cccd172..e26b8d2f8e 100644 --- a/azure-ipam/internal/buildversion/buildversion.go +++ b/azure-ipam/internal/buildinfo/buildinfo.go @@ -1,6 +1,6 @@ -package buildversion +package buildinfo // this will be populate by the Go compiler via // the -ldflags, which insert dynamic information // into the binary at build time -var BuildVersion string +var Version string diff --git a/azure-ipam/ipam.go b/azure-ipam/ipam.go index b45114f38c..ed0101bbb3 100644 --- a/azure-ipam/ipam.go +++ b/azure-ipam/ipam.go @@ -7,7 +7,7 @@ import ( "io" "net" - "github.com/Azure/azure-container-networking/azure-ipam/internal/buildversion" + "github.com/Azure/azure-container-networking/azure-ipam/internal/buildinfo" "github.com/Azure/azure-container-networking/azure-ipam/ipconfig" "github.com/Azure/azure-container-networking/cns" cnscli "github.com/Azure/azure-container-networking/cns/client" @@ -43,7 +43,7 @@ type cnsClient interface { func NewPlugin(logger *zap.Logger, c cnsClient, out io.Writer) (*IPAMPlugin, error) { plugin := &IPAMPlugin{ Name: pluginName, - Version: buildversion.BuildVersion, + Version: buildinfo.Version, logger: logger, out: out, cnsClient: c, From 29b6f67ccc867f8439746197b48e2c0bd2732e83 Mon Sep 17 00:00:00 2001 From: Quang Nguyen Date: Wed, 4 Oct 2023 12:55:13 +0000 Subject: [PATCH 53/85] refractor: fix conflicts --- cns/api.go | 13 ------- cns/middlewares/multitenant.go | 65 ---------------------------------- cns/restserver/ipam.go | 28 --------------- cns/restserver/restserver.go | 9 ----- cns/service/main.go | 1 + 5 files changed, 1 insertion(+), 115 deletions(-) delete mode 100644 cns/middlewares/multitenant.go diff --git a/cns/api.go b/cns/api.go index a98084b858..19f8a63da8 100644 --- a/cns/api.go +++ b/cns/api.go @@ -48,7 +48,6 @@ type HTTPService interface { GetPendingReleaseIPConfigs() []IPConfigurationStatus GetPodIPConfigState() map[string]IPConfigurationStatus MarkIPAsPendingRelease(numberToMark int) (map[string]IPConfigurationStatus, error) -<<<<<<< HEAD AttachSWIFTv2Middleware(middleware SWIFTv2Middleware) } @@ -60,18 +59,6 @@ type SWIFTv2Middleware interface { } type IPConfigsRequestValidator func(context.Context, *IPConfigsRequest) (types.ResponseCode, string) -======= - AttachMultitenantMiddleware(middleware MultitenantMiddleware) -} - -// Middleware interface for testing later on -type MultitenantMiddleware interface { - Validator() IPConfigValidator - GetMultitenantIPConfig(podInfo PodInfo) (*PodIpInfo, error) -} - -type IPConfigValidator func(ipConfigsRequest *IPConfigsRequest) (types.ResponseCode, string) ->>>>>>> 9e461501 (pod client placeholder) // This is used for KubernetesCRD orchestrator Type where NC has multiple ips. // This struct captures the state for SecondaryIPs associated to a given NC diff --git a/cns/middlewares/multitenant.go b/cns/middlewares/multitenant.go deleted file mode 100644 index fff83f75e7..0000000000 --- a/cns/middlewares/multitenant.go +++ /dev/null @@ -1,65 +0,0 @@ -package middlewares - -import ( - "context" - "fmt" - - "github.com/Azure/azure-container-networking/cns" - "github.com/Azure/azure-container-networking/cns/configuration" - "github.com/Azure/azure-container-networking/cns/types" - v1 "k8s.io/api/core/v1" - k8types "k8s.io/apimachinery/pkg/types" - "sigs.k8s.io/controller-runtime/pkg/client" -) - -type MultitenantMiddleware struct { - // TODO: implement - // need cached scoped client for pods - // need client for MTPNC CRD for x-ref pods - cli client.Client -} - -func NewMultitenantMiddleware(cli client.Client) *MultitenantMiddleware { - return &MultitenantMiddleware{ - cli: cli, - } -} - -// Return the validator function for the middleware -func (m *MultitenantMiddleware) Validator() cns.IPConfigValidator { - return m.validateMultitenantIPConfigsRequest -} - -// validateMultitenantIPConfigsRequest validates if pod is multitenant -// nolint -func (m *MultitenantMiddleware) validateMultitenantIPConfigsRequest(req *cns.IPConfigsRequest) (respCode types.ResponseCode, message string) { - // Retrieve the pod from the cluster - podInfo, err := cns.UnmarshalPodInfo(req.OrchestratorContext) - if err != nil { - errBuf := fmt.Sprintf("unmarshalling pod info from ipconfigs request %v failed with error %v", req, err) - return types.UnexpectedError, errBuf - } - podNamespacedName := k8types.NamespacedName{Namespace: podInfo.Namespace(), Name: podInfo.Name()} - pod := v1.Pod{} - err = m.cli.Get(context.TODO(), podNamespacedName, &pod) - if err != nil { - errBuf := fmt.Sprintf("failed to get pod %v with error %v", podNamespacedName, err) - return types.UnexpectedError, errBuf - } - - // check the pod labels for Swift V2, enrich the request with the multitenant flag. TBD on the label - if _, ok := pod.Labels[configuration.LabelSwiftV2]; ok { - req.Multitenant = true - } - return types.Success, "" -} - -// nolint -// GetMultitenantIPConfig returns the IP config for a multitenant pod from the MTPNC CRD -func (m *MultitenantMiddleware) GetMultitenantIPConfig(podInfo cns.PodInfo) (*cns.PodIpInfo, error) { - /** - TODO: - - Check if the MTPNC CRD exists for the pod, if not, return error - **/ - return nil, nil -} diff --git a/cns/restserver/ipam.go b/cns/restserver/ipam.go index 675bfe7b73..2e51ef6bde 100644 --- a/cns/restserver/ipam.go +++ b/cns/restserver/ipam.go @@ -37,34 +37,6 @@ func (service *HTTPRestService) requestIPConfigHandlerHelper(ctx context.Context }, errors.New("failed to validate ip config request") } - var MTNpodIPInfo *cns.PodIpInfo - - // Request is for multitenant pod - if ipconfigsRequest.Multitenant { - // Multitenant middleware not set - if service.MultitenantMiddleware == nil { - return &cns.IPConfigsResponse{ - Response: cns.Response{ - ReturnCode: types.FailedToAllocateIPConfig, - Message: fmt.Sprintf("AllocateIPConfig failed: %v, IP config request is %v", errors.New("request is for multitenant pod but multitenant middleware is nil"), ipconfigsRequest), - }, - PodIPInfo: []cns.PodIpInfo{}, - }, errors.New("request is for multitenant pod but multitenant middleware is nil") - } - // If pod is multitenant and we failed to grab its MTPNC IP config, return error immediately - podIPInfo, err := service.MultitenantMiddleware.GetMultitenantIPConfig(podInfo) - if err != nil { - return &cns.IPConfigsResponse{ - Response: cns.Response{ - ReturnCode: types.FailedToAllocateIPConfig, - Message: fmt.Sprintf("AllocateIPConfig failed: %v, IP config request is %v", err, ipconfigsRequest), - }, - PodIPInfo: []cns.PodIpInfo{}, - }, errors.Wrapf(err, "failed to get multitenant IP config %v", ipconfigsRequest) - } - MTNpodIPInfo = podIPInfo - } - // record a pod requesting an IP service.podsPendingIPAssignment.Push(podInfo.Key()) diff --git a/cns/restserver/restserver.go b/cns/restserver/restserver.go index 689f4d2d95..08fff69664 100644 --- a/cns/restserver/restserver.go +++ b/cns/restserver/restserver.go @@ -228,17 +228,8 @@ func (service *HTTPRestService) Init(config *common.ServiceConfig) error { return err } -<<<<<<< HEAD // Adding the default ipconfigs request validator service.ipConfigsRequestValidators = []cns.IPConfigsRequestValidator{service.validateDefaultIPConfigsRequest} -======= - // Adding ipConfigsValidators -<<<<<<< HEAD - service.ipConfigsValidators = []middlewares.IPConfigValidator{service.validateDefaultIPConfigsRequest} ->>>>>>> ab16c636 (update IPAM branching) -======= - service.ipConfigsValidators = []cns.IPConfigValidator{service.validateDefaultIPConfigsRequest} ->>>>>>> 9e461501 (pod client placeholder) // Add handlers. listener := service.Listener diff --git a/cns/service/main.go b/cns/service/main.go index 0734773cef..79cd98f2a6 100644 --- a/cns/service/main.go +++ b/cns/service/main.go @@ -37,6 +37,7 @@ import ( nncctrl "github.com/Azure/azure-container-networking/cns/kubecontroller/nodenetworkconfig" podctrl "github.com/Azure/azure-container-networking/cns/kubecontroller/pod" "github.com/Azure/azure-container-networking/cns/logger" + "github.com/Azure/azure-container-networking/cns/middlewares" "github.com/Azure/azure-container-networking/cns/multitenantcontroller" "github.com/Azure/azure-container-networking/cns/multitenantcontroller/multitenantoperator" "github.com/Azure/azure-container-networking/cns/restserver" From c79a0e678746aed6853069e110d20214901c5d38 Mon Sep 17 00:00:00 2001 From: Quang Nguyen Date: Wed, 4 Oct 2023 13:02:55 +0000 Subject: [PATCH 54/85] refractor: revert podwatcher code changes --- cns/service/main.go | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/cns/service/main.go b/cns/service/main.go index 79cd98f2a6..6bd6b44cea 100644 --- a/cns/service/main.go +++ b/cns/service/main.go @@ -1253,23 +1253,9 @@ func InitializeCRDState(ctx context.Context, httpRestService cns.HTTPService, cn "kube-system": {FieldSelector: fields.SelectorFromSet(fields.Set{"metadata.name": nodeName})}, }, }, - &corev1.Pod{}: { // nolint: typecheck - Field: fields.SelectorFromSet(fields.Set{"spec.nodeName": nodeName}), - }, }, } - crdSchemes := kuberuntime.NewScheme() - if err = v1alpha.AddToScheme(crdSchemes); err != nil { - return errors.Wrap(err, "failed to add nodenetworkconfig/v1alpha to scheme") - } - if err = v1alpha.AddToScheme(crdSchemes); err != nil { - return errors.Wrap(err, "failed to add clustersubnetstate/v1alpha1 to scheme") - } - if err = corev1.AddToScheme(crdSchemes); err != nil { - return errors.Wrap(err, "failed to add core/v1 to scheme") - } - if cnsconfig.WatchPods { cacheOpts.ByObject[&corev1.Pod{}] = cache.ByObject{ Field: fields.SelectorFromSet(fields.Set{"spec.nodeName": nodeName}), From ff291a2b6af01d504715e0810ab0985ed806add4 Mon Sep 17 00:00:00 2001 From: Quang Nguyen Date: Wed, 4 Oct 2023 15:32:16 +0000 Subject: [PATCH 55/85] docs: change comment --- cns/restserver/ipam_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cns/restserver/ipam_test.go b/cns/restserver/ipam_test.go index 56ef89dcc3..9b17b79393 100644 --- a/cns/restserver/ipam_test.go +++ b/cns/restserver/ipam_test.go @@ -1583,7 +1583,7 @@ func TestIPAMGetSWIFTv2IP(t *testing.T) { t.Fatalf("Expected to get 3 pod IP info (IPv4, IPv6, Multitenant IP), actual %d", len(podIPInfo)) } - // Asserting that multitenant IP is returned + // Asserting that SWIFT v2 IP is returned assert.Equal(t, SWIFTv2IP, podIPInfo[2].PodIPConfig.IPAddress) assert.Equal(t, SWIFTv2MAC, podIPInfo[2].MacAddress) assert.Equal(t, cns.DelegatedVMNIC, podIPInfo[2].NICType) From 17702e951fba13a2d2146b8bbddfd0e897eaef19 Mon Sep 17 00:00:00 2001 From: Quang Nguyen Date: Wed, 4 Oct 2023 22:59:19 +0000 Subject: [PATCH 56/85] refractor: change CIDR to CDIRs --- cns/configuration/env.go | 32 ++++++++++++++++---------------- cns/configuration/env_test.go | 16 ++++++++-------- cns/middlewares/swiftV2.go | 4 ++-- cns/middlewares/swiftV2_test.go | 6 +++--- 4 files changed, 29 insertions(+), 29 deletions(-) diff --git a/cns/configuration/env.go b/cns/configuration/env.go index 85d00b4c5e..12abfd594f 100644 --- a/cns/configuration/env.go +++ b/cns/configuration/env.go @@ -14,8 +14,8 @@ const ( // LabelNodeSwiftV2 is the Node label for Swift V2 LabelNodeSwiftV2 = "kubernetes.azure.com/podnetwork-multi-tenancy-enabled" LabelPodSwiftV2 = "kubernetes.azure.com/pod-network" - EnvPodV4CIDR = "POD_V4_CIDR" - EnvPodV6CIDR = "POD_V6_CIDR" + EnvPodV4CIDRs = "POD_V4_CIDRs" + EnvPodV6CIDRs = "POD_V6_CIDRs" EnvServiceV4CIDR = "SERVICE_V4_CIDR" EnvServiceV6CIDR = "SERVICE_V6_CIDR" ) @@ -23,11 +23,11 @@ const ( // ErrNodeNameUnset indicates the the $EnvNodeName variable is unset in the environment. var ErrNodeNameUnset = errors.Errorf("must declare %s environment variable", EnvNodeName) -// ErrPodV4CIDRUnset indicates the the $EnvPodV4CIDR variable is unset in the environment. -var ErrPodV4CIDRUnset = errors.Errorf("must declare %s environment variable", EnvPodV4CIDR) +// ErrPodV4CIDRsUnset indicates the the $EnvPodV4CIDRs variable is unset in the environment. +var ErrPodV4CIDRsUnset = errors.Errorf("must declare %s environment variable", EnvPodV4CIDRs) -// ErrPodV6CIDRUnset indicates the the $EnvPodV6CIDR variable is unset in the environment. -var ErrPodV6CIDRUnset = errors.Errorf("must declare %s environment variable", EnvPodV6CIDR) +// ErrPodV6CIDRsUnset indicates the the $EnvPodV6CIDRs variable is unset in the environment. +var ErrPodV6CIDRsUnset = errors.Errorf("must declare %s environment variable", EnvPodV6CIDRs) // ErrServiceV4CIDRUnset indicates the the $EnvServiceV4CIDR variable is unset in the environment. var ErrServiceV4CIDRUnset = errors.Errorf("must declare %s environment variable", EnvServiceV4CIDR) @@ -49,20 +49,20 @@ func NodeIP() string { return os.Getenv(EnvNodeIP) } -func PodV4CIDR() (string, error) { - podCIDRv4 := os.Getenv(EnvPodV4CIDR) - if podCIDRv4 == "" { - return "", ErrPodV4CIDRUnset +func PodV4CIDRs() (string, error) { + podCIDRsv4 := os.Getenv(EnvPodV4CIDRs) + if podCIDRsv4 == "" { + return "", ErrPodV4CIDRsUnset } - return podCIDRv4, nil + return podCIDRsv4, nil } -func PodV6CIDR() (string, error) { - podCIDRv6 := os.Getenv(EnvPodV6CIDR) - if podCIDRv6 == "" { - return "", ErrPodV6CIDRUnset +func PodV6CIDRs() (string, error) { + podCIDRsv6 := os.Getenv(EnvPodV6CIDRs) + if podCIDRsv6 == "" { + return "", ErrPodV6CIDRsUnset } - return podCIDRv6, nil + return podCIDRsv6, nil } func ServiceV4CIDR() (string, error) { diff --git a/cns/configuration/env_test.go b/cns/configuration/env_test.go index afebcf8ab4..eb16911d9c 100644 --- a/cns/configuration/env_test.go +++ b/cns/configuration/env_test.go @@ -19,21 +19,21 @@ func TestNodeName(t *testing.T) { } func TestPodV4CIDR(t *testing.T) { - _, err := PodV4CIDR() + _, err := PodV4CIDRs() require.Error(t, err) - require.ErrorIs(t, err, ErrPodV4CIDRUnset) - os.Setenv(EnvPodV4CIDR, "test") - cidr, err := PodV4CIDR() + require.ErrorIs(t, err, ErrPodV4CIDRsUnset) + os.Setenv(EnvPodV4CIDRs, "test") + cidr, err := PodV4CIDRs() assert.NoError(t, err) assert.Equal(t, "test", cidr) } func TestPodV6CIDR(t *testing.T) { - _, err := PodV6CIDR() + _, err := PodV6CIDRs() require.Error(t, err) - require.ErrorIs(t, err, ErrPodV6CIDRUnset) - os.Setenv(EnvPodV6CIDR, "test") - cidr, err := PodV6CIDR() + require.ErrorIs(t, err, ErrPodV6CIDRsUnset) + os.Setenv(EnvPodV6CIDRs, "test") + cidr, err := PodV6CIDRs() assert.NoError(t, err) assert.Equal(t, "test", cidr) } diff --git a/cns/middlewares/swiftV2.go b/cns/middlewares/swiftV2.go index 5855b07adb..3939b32f7d 100644 --- a/cns/middlewares/swiftV2.go +++ b/cns/middlewares/swiftV2.go @@ -93,7 +93,7 @@ func (m *SWIFTv2Middleware) SetRoutes(podIPInfo *cns.PodIpInfo) error { // Check if IP is v4 or v6 if net.ParseIP(podIPInfo.PodIPConfig.IPAddress).To4() != nil { // route for IPv4 podCIDR traffic - podCIDRv4, err := configuration.PodV4CIDR() + podCIDRv4, err := configuration.PodV4CIDRs() if err != nil { return fmt.Errorf("failed to get podCIDRv4 from env : %w", err) } @@ -104,7 +104,7 @@ func (m *SWIFTv2Middleware) SetRoutes(podIPInfo *cns.PodIpInfo) error { podIPInfo.Routes = []cns.Route{podCIDRv4Route} } else { // route for IPv6 podCIDR traffic - podCIDRv6, err := configuration.PodV6CIDR() + podCIDRv6, err := configuration.PodV6CIDRs() if err != nil { return fmt.Errorf("failed to get podCIDRv6 from env : %w", err) } diff --git a/cns/middlewares/swiftV2_test.go b/cns/middlewares/swiftV2_test.go index e18ad0db65..e2f9eb8d98 100644 --- a/cns/middlewares/swiftV2_test.go +++ b/cns/middlewares/swiftV2_test.go @@ -64,7 +64,7 @@ func TestValidateMultitenantIPConfigsRequestFailure(t *testing.T) { } func TestGetSWIFTv2IPConfigSuccess(t *testing.T) { - os.Setenv(configuration.EnvPodV4CIDR, "10.0.1.10/24") + os.Setenv(configuration.EnvPodV4CIDRs, "10.0.1.10/24") os.Setenv(configuration.EnvServiceV4CIDR, "10.0.2.10/24") middleware := SWIFTv2Middleware{Cli: mock.NewMockClient()} @@ -89,8 +89,8 @@ func TestGetSWIFTv2IPConfigFailure(t *testing.T) { func TestSetRoutesSuccess(t *testing.T) { middleware := SWIFTv2Middleware{Cli: mock.NewMockClient()} - os.Setenv(configuration.EnvPodV4CIDR, "10.0.1.10/24") - os.Setenv(configuration.EnvPodV6CIDR, "16A0:0010:AB00:001E::2/32") + os.Setenv(configuration.EnvPodV4CIDRs, "10.0.1.10/24") + os.Setenv(configuration.EnvPodV6CIDRs, "16A0:0010:AB00:001E::2/32") podIPInfo := []cns.PodIpInfo{ { PodIPConfig: cns.IPSubnet{ From 0e883769c01eb2370af0d72251101026a87034b9 Mon Sep 17 00:00:00 2001 From: Quang Nguyen Date: Fri, 6 Oct 2023 16:38:34 +0000 Subject: [PATCH 57/85] refractor: parse CIDRs as semicolons separated string from env in SetRoutes --- cns/configuration/env.go | 56 +++++++--------------- cns/configuration/env_test.go | 40 ++++------------ cns/middlewares/swiftV2.go | 82 ++++++++++++++++++++++++++------- cns/middlewares/swiftV2_test.go | 16 +++++-- 4 files changed, 103 insertions(+), 91 deletions(-) diff --git a/cns/configuration/env.go b/cns/configuration/env.go index 12abfd594f..59290df6fc 100644 --- a/cns/configuration/env.go +++ b/cns/configuration/env.go @@ -14,26 +14,18 @@ const ( // LabelNodeSwiftV2 is the Node label for Swift V2 LabelNodeSwiftV2 = "kubernetes.azure.com/podnetwork-multi-tenancy-enabled" LabelPodSwiftV2 = "kubernetes.azure.com/pod-network" - EnvPodV4CIDRs = "POD_V4_CIDRs" - EnvPodV6CIDRs = "POD_V6_CIDRs" - EnvServiceV4CIDR = "SERVICE_V4_CIDR" - EnvServiceV6CIDR = "SERVICE_V6_CIDR" + EnvPodCIDRs = "POD_CIDRs" + EnvServiceCIDRs = "SERVICE_CIDRs" ) // ErrNodeNameUnset indicates the the $EnvNodeName variable is unset in the environment. var ErrNodeNameUnset = errors.Errorf("must declare %s environment variable", EnvNodeName) -// ErrPodV4CIDRsUnset indicates the the $EnvPodV4CIDRs variable is unset in the environment. -var ErrPodV4CIDRsUnset = errors.Errorf("must declare %s environment variable", EnvPodV4CIDRs) +// ErrPodCIDRsUnset indicates the the $EnvPodCIDRs variable is unset in the environment. +var ErrPodCIDRsUnset = errors.Errorf("must declare %s environment variable", EnvPodCIDRs) -// ErrPodV6CIDRsUnset indicates the the $EnvPodV6CIDRs variable is unset in the environment. -var ErrPodV6CIDRsUnset = errors.Errorf("must declare %s environment variable", EnvPodV6CIDRs) - -// ErrServiceV4CIDRUnset indicates the the $EnvServiceV4CIDR variable is unset in the environment. -var ErrServiceV4CIDRUnset = errors.Errorf("must declare %s environment variable", EnvServiceV4CIDR) - -// ErrServiceV6CIDRUnset indicates the the $EnvServiceV6CIDR variable is unset in the environment. -var ErrServiceV6CIDRUnset = errors.Errorf("must declare %s environment variable", EnvServiceV6CIDR) +// ErrServiceCIDRsUnset indicates the the $EnvServiceCIDRs variable is unset in the environment. +var ErrServiceCIDRsUnset = errors.Errorf("must declare %s environment variable", EnvServiceCIDRs) // NodeName checks the environment variables for the NODENAME and returns it or an error if unset. func NodeName() (string, error) { @@ -49,34 +41,18 @@ func NodeIP() string { return os.Getenv(EnvNodeIP) } -func PodV4CIDRs() (string, error) { - podCIDRsv4 := os.Getenv(EnvPodV4CIDRs) - if podCIDRsv4 == "" { - return "", ErrPodV4CIDRsUnset - } - return podCIDRsv4, nil -} - -func PodV6CIDRs() (string, error) { - podCIDRsv6 := os.Getenv(EnvPodV6CIDRs) - if podCIDRsv6 == "" { - return "", ErrPodV6CIDRsUnset - } - return podCIDRsv6, nil -} - -func ServiceV4CIDR() (string, error) { - serviceV4CIDR := os.Getenv(EnvServiceV4CIDR) - if serviceV4CIDR == "" { - return "", ErrServiceV4CIDRUnset +func PodCIDRs() (string, error) { + podCIDRs := os.Getenv(EnvPodCIDRs) + if podCIDRs == "" { + return "", ErrPodCIDRsUnset } - return serviceV4CIDR, nil + return podCIDRs, nil } -func ServiceV6CIDR() (string, error) { - serviceV6CIDR := os.Getenv(EnvServiceV6CIDR) - if serviceV6CIDR == "" { - return "", ErrServiceV6CIDRUnset +func ServiceCIDRs() (string, error) { + serviceCIDRs := os.Getenv(EnvServiceCIDRs) + if serviceCIDRs == "" { + return "", ErrServiceCIDRsUnset } - return serviceV6CIDR, nil + return serviceCIDRs, nil } diff --git a/cns/configuration/env_test.go b/cns/configuration/env_test.go index eb16911d9c..cc411ec583 100644 --- a/cns/configuration/env_test.go +++ b/cns/configuration/env_test.go @@ -18,42 +18,22 @@ func TestNodeName(t *testing.T) { assert.Equal(t, "test", name) } -func TestPodV4CIDR(t *testing.T) { - _, err := PodV4CIDRs() +func TestPodCIDRs(t *testing.T) { + _, err := PodCIDRs() require.Error(t, err) - require.ErrorIs(t, err, ErrPodV4CIDRsUnset) - os.Setenv(EnvPodV4CIDRs, "test") - cidr, err := PodV4CIDRs() + require.ErrorIs(t, err, ErrPodCIDRsUnset) + os.Setenv(EnvPodCIDRs, "test") + cidr, err := PodCIDRs() assert.NoError(t, err) assert.Equal(t, "test", cidr) } -func TestPodV6CIDR(t *testing.T) { - _, err := PodV6CIDRs() +func TestServiceCIDRs(t *testing.T) { + _, err := ServiceCIDRs() require.Error(t, err) - require.ErrorIs(t, err, ErrPodV6CIDRsUnset) - os.Setenv(EnvPodV6CIDRs, "test") - cidr, err := PodV6CIDRs() - assert.NoError(t, err) - assert.Equal(t, "test", cidr) -} - -func TestServiceV4CIDR(t *testing.T) { - _, err := ServiceV4CIDR() - require.Error(t, err) - require.ErrorIs(t, err, ErrServiceV4CIDRUnset) - os.Setenv(EnvServiceV4CIDR, "test") - cidr, err := ServiceV4CIDR() - assert.NoError(t, err) - assert.Equal(t, "test", cidr) -} - -func TestServiceV6CIDR(t *testing.T) { - _, err := ServiceV6CIDR() - require.Error(t, err) - require.ErrorIs(t, err, ErrServiceV6CIDRUnset) - os.Setenv(EnvServiceV6CIDR, "test") - cidr, err := ServiceV6CIDR() + require.ErrorIs(t, err, ErrServiceCIDRsUnset) + os.Setenv(EnvServiceCIDRs, "test") + cidr, err := ServiceCIDRs() assert.NoError(t, err) assert.Equal(t, "test", cidr) } diff --git a/cns/middlewares/swiftV2.go b/cns/middlewares/swiftV2.go index 3939b32f7d..9c5c292a59 100644 --- a/cns/middlewares/swiftV2.go +++ b/cns/middlewares/swiftV2.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "net" + "strings" "github.com/Azure/azure-container-networking/cns" "github.com/Azure/azure-container-networking/cns/configuration" @@ -82,6 +83,7 @@ func (m *SWIFTv2Middleware) GetIPConfig(ctx context.Context, podInfo cns.PodInfo // SetRoutes sets the routes for podIPInfo used in SWIFT V2 scenario. func (m *SWIFTv2Middleware) SetRoutes(podIPInfo *cns.PodIpInfo) error { + podIPInfo.Routes = []cns.Route{} switch podIPInfo.NICType { case cns.DelegatedVMNIC: // default route via SWIFT v2 interface @@ -90,32 +92,78 @@ func (m *SWIFTv2Middleware) SetRoutes(podIPInfo *cns.PodIpInfo) error { } podIPInfo.Routes = []cns.Route{route} case cns.InfraNIC: - // Check if IP is v4 or v6 + podCIDRs, err := configuration.PodCIDRs() + if err != nil { + return fmt.Errorf("failed to get podCIDRs from env : %w", err) + } + podCIDRsV4, podCIDRv6, err := parseCIDRs(podCIDRs) + if err != nil { + return fmt.Errorf("failed to parse podCIDRs : %w", err) + } + + serviceCIDRs, err := configuration.ServiceCIDRs() + if err != nil { + return fmt.Errorf("failed to get serviceCIDRs from env : %w", err) + } + serviceCIDRsV4, serviceCIDRsV6, err := parseCIDRs(serviceCIDRs) + if err != nil { + return fmt.Errorf("failed to parse serviceCIDRs : %w", err) + } + // Check if the podIPInfo is IPv4 or IPv6 if net.ParseIP(podIPInfo.PodIPConfig.IPAddress).To4() != nil { - // route for IPv4 podCIDR traffic - podCIDRv4, err := configuration.PodV4CIDRs() - if err != nil { - return fmt.Errorf("failed to get podCIDRv4 from env : %w", err) + // routes for IPv4 podCIDR traffic + for _, podCIDRv4 := range podCIDRsV4 { + podCIDRv4Route := cns.Route{ + IPAddress: podCIDRv4, + GatewayIPAddress: overlayGatewayv4, + } + podIPInfo.Routes = append(podIPInfo.Routes, podCIDRv4Route) } - podCIDRv4Route := cns.Route{ - IPAddress: podCIDRv4, - GatewayIPAddress: overlayGatewayv4, + // route for IPv4 serviceCIDR traffic + for _, serviceCIDRv4 := range serviceCIDRsV4 { + serviceCIDRv4Route := cns.Route{ + IPAddress: serviceCIDRv4, + GatewayIPAddress: overlayGatewayv4, + } + podIPInfo.Routes = append(podIPInfo.Routes, serviceCIDRv4Route) } - podIPInfo.Routes = []cns.Route{podCIDRv4Route} + } else { - // route for IPv6 podCIDR traffic - podCIDRv6, err := configuration.PodV6CIDRs() - if err != nil { - return fmt.Errorf("failed to get podCIDRv6 from env : %w", err) + // routes for IPv6 podCIDR traffic + for _, podCIDRv6 := range podCIDRv6 { + podCIDRv6Route := cns.Route{ + IPAddress: podCIDRv6, + GatewayIPAddress: overlayGatewayV6, + } + podIPInfo.Routes = append(podIPInfo.Routes, podCIDRv6Route) } - podCIDRv6Route := cns.Route{ - IPAddress: podCIDRv6, - GatewayIPAddress: overlayGatewayV6, + // route for IPv6 serviceCIDR traffic + for _, serviceCIDRv6 := range serviceCIDRsV6 { + serviceCIDRv6Route := cns.Route{ + IPAddress: serviceCIDRv6, + GatewayIPAddress: overlayGatewayV6, + } + podIPInfo.Routes = append(podIPInfo.Routes, serviceCIDRv6Route) } - podIPInfo.Routes = []cns.Route{podCIDRv6Route} } default: return ErrInvalidSWIFTv2NICType } return nil } + +// parseCIDRs parses the semicolons separated CIDRs string and returns the IPv4 and IPv6 CIDRs. +func parseCIDRs(cidrs string) (v4IPs, v6IPs []string, err error) { + for _, cidr := range strings.Split(cidrs, ";") { + ip, _, err := net.ParseCIDR(cidr) + if err != nil { + return nil, nil, fmt.Errorf("failed to parse cidr %s : %w", cidr, err) + } + if ip.To4() != nil { + v4IPs = append(v4IPs, cidr) + } else { + v6IPs = append(v6IPs, cidr) + } + } + return v4IPs, v6IPs, nil +} diff --git a/cns/middlewares/swiftV2_test.go b/cns/middlewares/swiftV2_test.go index e2f9eb8d98..d2ffd18d50 100644 --- a/cns/middlewares/swiftV2_test.go +++ b/cns/middlewares/swiftV2_test.go @@ -64,8 +64,8 @@ func TestValidateMultitenantIPConfigsRequestFailure(t *testing.T) { } func TestGetSWIFTv2IPConfigSuccess(t *testing.T) { - os.Setenv(configuration.EnvPodV4CIDRs, "10.0.1.10/24") - os.Setenv(configuration.EnvServiceV4CIDR, "10.0.2.10/24") + os.Setenv(configuration.EnvPodCIDRs, "10.0.1.10/24") + os.Setenv(configuration.EnvServiceCIDRs, "10.0.2.10/24") middleware := SWIFTv2Middleware{Cli: mock.NewMockClient()} @@ -89,8 +89,8 @@ func TestGetSWIFTv2IPConfigFailure(t *testing.T) { func TestSetRoutesSuccess(t *testing.T) { middleware := SWIFTv2Middleware{Cli: mock.NewMockClient()} - os.Setenv(configuration.EnvPodV4CIDRs, "10.0.1.10/24") - os.Setenv(configuration.EnvPodV6CIDRs, "16A0:0010:AB00:001E::2/32") + os.Setenv(configuration.EnvPodCIDRs, "10.0.1.10/24;16A0:0010:AB00:001E::2/32") + os.Setenv(configuration.EnvServiceCIDRs, "10.0.0.0/16;16A0:0010:AB00:0000::/32") podIPInfo := []cns.PodIpInfo{ { PodIPConfig: cns.IPSubnet{ @@ -127,6 +127,10 @@ func TestSetRoutesSuccess(t *testing.T) { IPAddress: "10.0.1.10/24", GatewayIPAddress: overlayGatewayv4, }, + { + IPAddress: "10.0.0.0/16", + GatewayIPAddress: overlayGatewayv4, + }, }, }, { @@ -140,6 +144,10 @@ func TestSetRoutesSuccess(t *testing.T) { IPAddress: "16A0:0010:AB00:001E::2/32", GatewayIPAddress: overlayGatewayV6, }, + { + IPAddress: "16A0:0010:AB00:0000::/32", + GatewayIPAddress: overlayGatewayV6, + }, }, }, { From e59220e46a37101b8b3ee20e58b5184d61204f64 Mon Sep 17 00:00:00 2001 From: Quang Nguyen Date: Tue, 10 Oct 2023 21:37:56 +0000 Subject: [PATCH 58/85] docs: add minor comment --- cns/configuration/env.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/cns/configuration/env.go b/cns/configuration/env.go index 59290df6fc..3d391ef579 100644 --- a/cns/configuration/env.go +++ b/cns/configuration/env.go @@ -13,9 +13,10 @@ const ( EnvNodeIP = "NODE_IP" // LabelNodeSwiftV2 is the Node label for Swift V2 LabelNodeSwiftV2 = "kubernetes.azure.com/podnetwork-multi-tenancy-enabled" - LabelPodSwiftV2 = "kubernetes.azure.com/pod-network" - EnvPodCIDRs = "POD_CIDRs" - EnvServiceCIDRs = "SERVICE_CIDRs" + // LabelPodSwiftV2 is the Pod label for Swift V2 + LabelPodSwiftV2 = "kubernetes.azure.com/pod-network" + EnvPodCIDRs = "POD_CIDRs" + EnvServiceCIDRs = "SERVICE_CIDRs" ) // ErrNodeNameUnset indicates the the $EnvNodeName variable is unset in the environment. From 627b1a02f32f8d1e69947e450e1442376ae5bcc4 Mon Sep 17 00:00:00 2001 From: Quang Nguyen Date: Mon, 16 Oct 2023 20:40:09 +0000 Subject: [PATCH 59/85] refractor: change separator for parsing CIDRs --- cns/middlewares/swiftV2.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cns/middlewares/swiftV2.go b/cns/middlewares/swiftV2.go index 9c5c292a59..cea9e2f118 100644 --- a/cns/middlewares/swiftV2.go +++ b/cns/middlewares/swiftV2.go @@ -154,7 +154,7 @@ func (m *SWIFTv2Middleware) SetRoutes(podIPInfo *cns.PodIpInfo) error { // parseCIDRs parses the semicolons separated CIDRs string and returns the IPv4 and IPv6 CIDRs. func parseCIDRs(cidrs string) (v4IPs, v6IPs []string, err error) { - for _, cidr := range strings.Split(cidrs, ";") { + for _, cidr := range strings.Split(cidrs, ",") { ip, _, err := net.ParseCIDR(cidr) if err != nil { return nil, nil, fmt.Errorf("failed to parse cidr %s : %w", cidr, err) From 6085da3ac3ab3ef7d983f7b28e2b27edb34bb453 Mon Sep 17 00:00:00 2001 From: Quang Nguyen Date: Tue, 17 Oct 2023 19:16:09 +0000 Subject: [PATCH 60/85] feat: add rbac roles --- cns/azure-cns.yaml | 74 +++++++++++++++++++++++++++------------------- 1 file changed, 44 insertions(+), 30 deletions(-) diff --git a/cns/azure-cns.yaml b/cns/azure-cns.yaml index 541c2a6d84..0fe636f986 100644 --- a/cns/azure-cns.yaml +++ b/cns/azure-cns.yaml @@ -10,21 +10,33 @@ metadata: namespace: kube-system name: nodeNetConfigEditor rules: -- apiGroups: ["acn.azure.com"] - resources: ["nodenetworkconfigs"] - verbs: ["get", "list", "watch", "patch", "update"] + - apiGroups: ["acn.azure.com"] + resources: ["nodenetworkconfigs"] + verbs: ["get", "list", "watch", "patch", "update"] + - apiGroups: ["acn.azure.com"] + resources: ["clustersubnetstates"] + verbs: ["get", "list", "watch"] + - apiGroups: ["multitenancy.acn.azure.com"] + resources: + [ + "multitenantpodnetworkconfigs", + "nodeinfo", + "podnetworkinstances", + "podnetworks", + ] + verbs: ["get", "list", "watch", "patch", "update"] --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: pod-reader-all-namespaces rules: -- apiGroups: [""] - resources: ["pods"] - verbs: ["get", "watch", "list"] -- apiGroups: [""] - resources: ["nodes"] - verbs: ["get"] + - apiGroups: [""] + resources: ["pods"] + verbs: ["get", "watch", "list"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get"] --- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding @@ -32,9 +44,9 @@ metadata: name: nodeNetConfigEditorRoleBinding namespace: kube-system subjects: -- kind: ServiceAccount - name: azure-cns - namespace: kube-system + - kind: ServiceAccount + name: azure-cns + namespace: kube-system roleRef: kind: Role name: nodeNetConfigEditor @@ -45,9 +57,9 @@ kind: ClusterRoleBinding metadata: name: pod-reader-all-namespaces-binding subjects: -- kind: ServiceAccount - name: azure-cns - namespace: kube-system + - kind: ServiceAccount + name: azure-cns + namespace: kube-system roleRef: kind: ClusterRole name: pod-reader-all-namespaces @@ -75,17 +87,17 @@ spec: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - - matchExpressions: - - key: kubernetes.azure.com/cluster - operator: Exists - - key: type - operator: NotIn - values: - - virtual-kubelet - - key: beta.kubernetes.io/os - operator: In - values: - - linux + - matchExpressions: + - key: kubernetes.azure.com/cluster + operator: Exists + - key: type + operator: NotIn + values: + - virtual-kubelet + - key: beta.kubernetes.io/os + operator: In + values: + - linux priorityClassName: system-node-critical tolerations: - key: CriticalAddonsOnly @@ -98,7 +110,8 @@ spec: - name: cns-container image: mcr.microsoft.com/containernetworking/azure-cns:v1.4.7 imagePullPolicy: IfNotPresent - args: [ "-c", "tcp://$(CNSIpAddress):$(CNSPort)", "-t", "$(CNSLogTarget)"] + args: + ["-c", "tcp://$(CNSIpAddress):$(CNSPort)", "-t", "$(CNSLogTarget)"] volumeMounts: - name: log mountPath: /var/log @@ -127,9 +140,9 @@ spec: value: /etc/azure-cns/cns_config.json - name: NODENAME valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName hostNetwork: true volumes: - name: azure-endpoints @@ -189,4 +202,5 @@ data: "ManageEndpointState": false, "ProgramSNATIPTables" : false } + # Toggle ManageEndpointState and ProgramSNATIPTables to true for delegated IPAM use case. From c96cc666cea0ce919d82c27f69042057335bcb15 Mon Sep 17 00:00:00 2001 From: Quang Nguyen Date: Tue, 17 Oct 2023 19:21:03 +0000 Subject: [PATCH 61/85] fix: gofumpt --- cni/plugin.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cni/plugin.go b/cni/plugin.go index 82a17f178a..1b6b85abeb 100644 --- a/cni/plugin.go +++ b/cni/plugin.go @@ -24,8 +24,10 @@ import ( "go.uber.org/zap" ) -var logger = log.CNILogger.With(zap.String("component", "cni-plugin")) -var storeLogger = log.CNILogger.With(zap.String("component", "cni-store")) +var ( + logger = log.CNILogger.With(zap.String("component", "cni-plugin")) + storeLogger = log.CNILogger.With(zap.String("component", "cni-store")) +) var errEmptyContent = errors.New("read content is zero bytes") From f4f2f8eb8b32aa7aa2e0834851418dbddbf3a478 Mon Sep 17 00:00:00 2001 From: Quang Nguyen Date: Tue, 17 Oct 2023 19:39:16 +0000 Subject: [PATCH 62/85] fix: update clusterrole --- cns/azure-cns.yaml | 37 +++++++++++++++++++++++++------------ 1 file changed, 25 insertions(+), 12 deletions(-) diff --git a/cns/azure-cns.yaml b/cns/azure-cns.yaml index 0fe636f986..3a8f650acd 100644 --- a/cns/azure-cns.yaml +++ b/cns/azure-cns.yaml @@ -5,25 +5,38 @@ metadata: namespace: kube-system --- apiVersion: rbac.authorization.k8s.io/v1 -kind: Role +kind: ClusterRole metadata: - namespace: kube-system - name: nodeNetConfigEditor + name: acn-multitenancy-editor rules: - - apiGroups: ["acn.azure.com"] - resources: ["nodenetworkconfigs"] - verbs: ["get", "list", "watch", "patch", "update"] - apiGroups: ["acn.azure.com"] resources: ["clustersubnetstates"] verbs: ["get", "list", "watch"] - apiGroups: ["multitenancy.acn.azure.com"] resources: - [ - "multitenantpodnetworkconfigs", - "nodeinfo", - "podnetworkinstances", - "podnetworks", - ] + ["multitenantpodnetworkconfigs", "podnetworkinstances", "podnetworks"] + verbs: ["get", "list", "watch", "patch", "update"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: acn-multitenancy-editor-binding +subjects: + - kind: ServiceAccount + name: azure-cns +roleRef: + kind: ClusterRole + name: acn-multitenancy-editor + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + namespace: kube-system + name: nodeNetConfigEditor +rules: + - apiGroups: ["acn.azure.com"] + resources: ["nodenetworkconfigs"] verbs: ["get", "list", "watch", "patch", "update"] --- apiVersion: rbac.authorization.k8s.io/v1 From 0b5582293c7545cd4148685974dccbaa57fe4dee Mon Sep 17 00:00:00 2001 From: Quang Nguyen Date: Tue, 17 Oct 2023 19:41:28 +0000 Subject: [PATCH 63/85] fix: add namespace to clusterrolebinding --- cns/azure-cns.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/cns/azure-cns.yaml b/cns/azure-cns.yaml index 3a8f650acd..36f9def465 100644 --- a/cns/azure-cns.yaml +++ b/cns/azure-cns.yaml @@ -24,6 +24,7 @@ metadata: subjects: - kind: ServiceAccount name: azure-cns + namespace: kube-system roleRef: kind: ClusterRole name: acn-multitenancy-editor From b7639ed88304a88b422b72e9089e103a76b7fd7c Mon Sep 17 00:00:00 2001 From: Quang Nguyen Date: Tue, 17 Oct 2023 19:45:26 +0000 Subject: [PATCH 64/85] fix: UT --- cns/middlewares/swiftV2_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cns/middlewares/swiftV2_test.go b/cns/middlewares/swiftV2_test.go index d2ffd18d50..77429e07c1 100644 --- a/cns/middlewares/swiftV2_test.go +++ b/cns/middlewares/swiftV2_test.go @@ -89,8 +89,8 @@ func TestGetSWIFTv2IPConfigFailure(t *testing.T) { func TestSetRoutesSuccess(t *testing.T) { middleware := SWIFTv2Middleware{Cli: mock.NewMockClient()} - os.Setenv(configuration.EnvPodCIDRs, "10.0.1.10/24;16A0:0010:AB00:001E::2/32") - os.Setenv(configuration.EnvServiceCIDRs, "10.0.0.0/16;16A0:0010:AB00:0000::/32") + os.Setenv(configuration.EnvPodCIDRs, "10.0.1.10/24,16A0:0010:AB00:001E::2/32") + os.Setenv(configuration.EnvServiceCIDRs, "10.0.0.0/16,16A0:0010:AB00:0000::/32") podIPInfo := []cns.PodIpInfo{ { PodIPConfig: cns.IPSubnet{ From 768a9de1801e1d029446814e3e48d9e427b8a50d Mon Sep 17 00:00:00 2001 From: Quang Nguyen Date: Wed, 18 Oct 2023 16:18:33 +0000 Subject: [PATCH 65/85] fix: add labels toswift v2 clusterrole --- cns/azure-cns.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cns/azure-cns.yaml b/cns/azure-cns.yaml index 36f9def465..771fff2989 100644 --- a/cns/azure-cns.yaml +++ b/cns/azure-cns.yaml @@ -8,6 +8,8 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: acn-multitenancy-editor +labels: + addonmanager.kubernetes.io/mode: Reconcile rules: - apiGroups: ["acn.azure.com"] resources: ["clustersubnetstates"] From 7e00999c614ebedb8a680ded151ee6a79d867025 Mon Sep 17 00:00:00 2001 From: Quang Nguyen Date: Tue, 24 Oct 2023 16:38:24 +0000 Subject: [PATCH 66/85] fix: release default ipconfig early if getting swiftv2 ipconfig failed --- cns/restserver/ipam.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/cns/restserver/ipam.go b/cns/restserver/ipam.go index 2e51ef6bde..2b9542ac90 100644 --- a/cns/restserver/ipam.go +++ b/cns/restserver/ipam.go @@ -78,6 +78,13 @@ func (service *HTTPRestService) requestIPConfigHandlerHelper(ctx context.Context // In the future, if we have multiple scenario with secondary interfaces, we can add a switch case here SWIFTv2PodIPInfo, err := service.SWIFTv2Middleware.GetIPConfig(ctx, podInfo) if err != nil { + defer func() { + logger.Errorf("failed to get SWIFTv2 IP config : %v. Releasing default IP config...", err) + _, err := service.releaseIPConfigHandlerHelper(ctx, ipconfigsRequest) + if err != nil { + logger.Errorf("failed to release default IP config %v", err) + } + }() return &cns.IPConfigsResponse{ Response: cns.Response{ ReturnCode: types.FailedToAllocateIPConfig, @@ -92,6 +99,13 @@ func (service *HTTPRestService) requestIPConfigHandlerHelper(ctx context.Context ipInfo := &podIPInfo[i] err := service.SWIFTv2Middleware.SetRoutes(ipInfo) if err != nil { + defer func() { + logger.Errorf("failed to set routes for SWIFTv2 IP config : %v. Releasing default IP config...", err) + _, err := service.releaseIPConfigHandlerHelper(ctx, ipconfigsRequest) + if err != nil { + logger.Errorf("failed to release default IP config %v", err) + } + }() return &cns.IPConfigsResponse{ Response: cns.Response{ ReturnCode: types.FailedToAllocateIPConfig, From 7d791f3e98d6219b2573427aab7b48d70bc990f0 Mon Sep 17 00:00:00 2001 From: Quang Nguyen Date: Tue, 24 Oct 2023 17:45:21 +0000 Subject: [PATCH 67/85] test: add more UT --- cns/middlewares/mock/mockSWIFTv2.go | 124 ++++++++++++++++++++++++++-- cns/middlewares/swiftV2.go | 8 +- cns/middlewares/swiftV2_test.go | 48 +++++++++-- cns/restserver/ipam.go | 6 +- cns/restserver/ipam_test.go | 68 ++++++++++++++- 5 files changed, 233 insertions(+), 21 deletions(-) diff --git a/cns/middlewares/mock/mockSWIFTv2.go b/cns/middlewares/mock/mockSWIFTv2.go index dd0966c76e..c2106e3d58 100644 --- a/cns/middlewares/mock/mockSWIFTv2.go +++ b/cns/middlewares/mock/mockSWIFTv2.go @@ -4,6 +4,9 @@ import ( "context" "errors" "fmt" + "net" + "os" + "strings" "github.com/Azure/azure-container-networking/cns" "github.com/Azure/azure-container-networking/cns/configuration" @@ -14,8 +17,15 @@ import ( ) var ( - errMTPNCNotReady = errors.New("mtpnc is not ready") - errFailedToGetPod = errors.New("failed to get pod") + errMTPNCNotReady = errors.New("mtpnc is not ready") + errFailedToGetPod = errors.New("failed to get pod") + errInvalidSWIFTv2NICType = errors.New("invalid NIC type for SWIFT v2 scenario") +) + +const ( + prefixLength = 32 + overlayGatewayv4 = "169.254.1.1" + overlayGatewayV6 = "fe80::1234:5678:9abc" ) type MockSWIFTv2Middleware struct { @@ -29,10 +39,6 @@ func NewMockSWIFTv2Middleware() *MockSWIFTv2Middleware { testPod1.Labels[configuration.LabelPodSwiftV2] = "true" testMTPNC1 := v1alpha1.MultitenantPodNetworkConfig{} - testMTPNC1.Status.PrimaryIP = "192.168.0.1" - testMTPNC1.Status.MacAddress = "00:00:00:00:00:00" - testMTPNC1.Status.GatewayIP = "10.0.0.1" - testMTPNC1.Status.NCID = "testncid" return &MockSWIFTv2Middleware{ mtPodState: map[string]*v1.Pod{"testpod1namespace/testpod1": &testPod1}, @@ -40,6 +46,28 @@ func NewMockSWIFTv2Middleware() *MockSWIFTv2Middleware { } } +func (m *MockSWIFTv2Middleware) SetMTPNCReady() { + m.mtpncState["testpod1namespace/testpod1"].Status.PrimaryIP = "192.168.0.1" + m.mtpncState["testpod1namespace/testpod1"].Status.MacAddress = "00:00:00:00:00:00" + m.mtpncState["testpod1namespace/testpod1"].Status.GatewayIP = "10.0.0.1" + m.mtpncState["testpod1namespace/testpod1"].Status.NCID = "testncid" +} + +func (m *MockSWIFTv2Middleware) SetEnvVar() { + os.Setenv(configuration.EnvPodCIDRs, "10.0.1.10/24") + os.Setenv(configuration.EnvServiceCIDRs, "10.0.2.10/24") +} + +func (m *MockSWIFTv2Middleware) UnsetEnvVar() error { + if err := os.Unsetenv(configuration.EnvPodCIDRs); err != nil { + return fmt.Errorf("failed to unset env var %s : %w", configuration.EnvPodCIDRs, err) + } + if err := os.Unsetenv(configuration.EnvServiceCIDRs); err != nil { + return fmt.Errorf("failed to unset env var %s : %w", configuration.EnvServiceCIDRs, err) + } + return nil +} + // validateMultitenantIPConfigsRequest validates if pod is multitenant // nolint func (m *MockSWIFTv2Middleware) ValidateIPConfigsRequest(_ context.Context, req *cns.IPConfigsRequest) (respCode types.ResponseCode, message string) { @@ -87,6 +115,88 @@ func (m *MockSWIFTv2Middleware) GetIPConfig(_ context.Context, podInfo cns.PodIn return podIPInfo, nil } -func (m *MockSWIFTv2Middleware) SetRoutes(_ *cns.PodIpInfo) error { +func (m *MockSWIFTv2Middleware) SetRoutes(podIPInfo *cns.PodIpInfo) error { + podIPInfo.Routes = []cns.Route{} + switch podIPInfo.NICType { + case cns.DelegatedVMNIC: + // default route via SWIFT v2 interface + route := cns.Route{ + IPAddress: "0.0.0.0/0", + } + podIPInfo.Routes = []cns.Route{route} + case cns.InfraNIC: + podCIDRs, err := configuration.PodCIDRs() + if err != nil { + return fmt.Errorf("failed to get podCIDRs from env : %w", err) + } + podCIDRsV4, podCIDRv6, err := parseCIDRs(podCIDRs) + if err != nil { + return fmt.Errorf("failed to parse podCIDRs : %w", err) + } + + serviceCIDRs, err := configuration.ServiceCIDRs() + if err != nil { + return fmt.Errorf("failed to get serviceCIDRs from env : %w", err) + } + serviceCIDRsV4, serviceCIDRsV6, err := parseCIDRs(serviceCIDRs) + if err != nil { + return fmt.Errorf("failed to parse serviceCIDRs : %w", err) + } + // Check if the podIPInfo is IPv4 or IPv6 + if net.ParseIP(podIPInfo.PodIPConfig.IPAddress).To4() != nil { + // routes for IPv4 podCIDR traffic + for _, podCIDRv4 := range podCIDRsV4 { + podCIDRv4Route := cns.Route{ + IPAddress: podCIDRv4, + GatewayIPAddress: overlayGatewayv4, + } + podIPInfo.Routes = append(podIPInfo.Routes, podCIDRv4Route) + } + // route for IPv4 serviceCIDR traffic + for _, serviceCIDRv4 := range serviceCIDRsV4 { + serviceCIDRv4Route := cns.Route{ + IPAddress: serviceCIDRv4, + GatewayIPAddress: overlayGatewayv4, + } + podIPInfo.Routes = append(podIPInfo.Routes, serviceCIDRv4Route) + } + + } else { + // routes for IPv6 podCIDR traffic + for _, podCIDRv6 := range podCIDRv6 { + podCIDRv6Route := cns.Route{ + IPAddress: podCIDRv6, + GatewayIPAddress: overlayGatewayV6, + } + podIPInfo.Routes = append(podIPInfo.Routes, podCIDRv6Route) + } + // route for IPv6 serviceCIDR traffic + for _, serviceCIDRv6 := range serviceCIDRsV6 { + serviceCIDRv6Route := cns.Route{ + IPAddress: serviceCIDRv6, + GatewayIPAddress: overlayGatewayV6, + } + podIPInfo.Routes = append(podIPInfo.Routes, serviceCIDRv6Route) + } + } + default: + return errInvalidSWIFTv2NICType + } return nil } + +// parseCIDRs parses the semicolons separated CIDRs string and returns the IPv4 and IPv6 CIDRs. +func parseCIDRs(cidrs string) (v4IPs, v6IPs []string, err error) { + for _, cidr := range strings.Split(cidrs, ",") { + ip, _, err := net.ParseCIDR(cidr) + if err != nil { + return nil, nil, fmt.Errorf("failed to parse cidr %s : %w", cidr, err) + } + if ip.To4() != nil { + v4IPs = append(v4IPs, cidr) + } else { + v6IPs = append(v6IPs, cidr) + } + } + return v4IPs, v6IPs, nil +} diff --git a/cns/middlewares/swiftV2.go b/cns/middlewares/swiftV2.go index cea9e2f118..68524a74cb 100644 --- a/cns/middlewares/swiftV2.go +++ b/cns/middlewares/swiftV2.go @@ -17,8 +17,8 @@ import ( ) var ( - ErrMTPNCNotReady = errors.New("mtpnc is not ready") - ErrInvalidSWIFTv2NICType = errors.New("invalid NIC type for SWIFT v2 scenario") + errMTPNCNotReady = errors.New("mtpnc is not ready") + errInvalidSWIFTv2NICType = errors.New("invalid NIC type for SWIFT v2 scenario") ) const ( @@ -65,7 +65,7 @@ func (m *SWIFTv2Middleware) GetIPConfig(ctx context.Context, podInfo cns.PodInfo // Check if the MTPNC CRD is ready. If one of the fields is empty, return error if mtpnc.Status.PrimaryIP == "" || mtpnc.Status.MacAddress == "" || mtpnc.Status.NCID == "" || mtpnc.Status.GatewayIP == "" { - return cns.PodIpInfo{}, ErrMTPNCNotReady + return cns.PodIpInfo{}, errMTPNCNotReady } podIPInfo := cns.PodIpInfo{ PodIPConfig: cns.IPSubnet{ @@ -147,7 +147,7 @@ func (m *SWIFTv2Middleware) SetRoutes(podIPInfo *cns.PodIpInfo) error { } } default: - return ErrInvalidSWIFTv2NICType + return errInvalidSWIFTv2NICType } return nil } diff --git a/cns/middlewares/swiftV2_test.go b/cns/middlewares/swiftV2_test.go index 77429e07c1..59eb5ebf6e 100644 --- a/cns/middlewares/swiftV2_test.go +++ b/cns/middlewares/swiftV2_test.go @@ -23,6 +23,16 @@ var ( testPod3Info = cns.NewPodInfo("718e04-eth0", testPod3GUID, "testpod3", "testpod3namespace") ) +func setEnvVar() { + os.Setenv(configuration.EnvPodCIDRs, "10.0.1.10/24,16A0:0010:AB00:001E::2/32") + os.Setenv(configuration.EnvServiceCIDRs, "10.0.0.0/16,16A0:0010:AB00:0000::/32") +} + +func unsetEnvVar() { + os.Unsetenv(configuration.EnvPodCIDRs) + os.Unsetenv(configuration.EnvServiceCIDRs) +} + func TestValidateMultitenantIPConfigsRequestSuccess(t *testing.T) { middleware := SWIFTv2Middleware{Cli: mock.NewMockClient()} @@ -64,8 +74,8 @@ func TestValidateMultitenantIPConfigsRequestFailure(t *testing.T) { } func TestGetSWIFTv2IPConfigSuccess(t *testing.T) { - os.Setenv(configuration.EnvPodCIDRs, "10.0.1.10/24") - os.Setenv(configuration.EnvServiceCIDRs, "10.0.2.10/24") + setEnvVar() + defer unsetEnvVar() middleware := SWIFTv2Middleware{Cli: mock.NewMockClient()} @@ -84,13 +94,13 @@ func TestGetSWIFTv2IPConfigFailure(t *testing.T) { // Pod's MTPNC is not ready test _, err = middleware.GetIPConfig(context.TODO(), testPod3Info) - assert.Error(t, err, ErrMTPNCNotReady.Error()) + assert.Error(t, err, errMTPNCNotReady.Error()) } func TestSetRoutesSuccess(t *testing.T) { middleware := SWIFTv2Middleware{Cli: mock.NewMockClient()} - os.Setenv(configuration.EnvPodCIDRs, "10.0.1.10/24,16A0:0010:AB00:001E::2/32") - os.Setenv(configuration.EnvServiceCIDRs, "10.0.0.0/16,16A0:0010:AB00:0000::/32") + setEnvVar() + defer unsetEnvVar() podIPInfo := []cns.PodIpInfo{ { PodIPConfig: cns.IPSubnet{ @@ -173,3 +183,31 @@ func TestSetRoutesSuccess(t *testing.T) { assert.DeepEqual(t, podIPInfo[i].Routes, desiredPodIPInfo[i].Routes) } } + +func TestSetRoutesFailure(t *testing.T) { + // Failure due to env var not set + middleware := SWIFTv2Middleware{Cli: mock.NewMockClient()} + podIPInfo := []cns.PodIpInfo{ + { + PodIPConfig: cns.IPSubnet{ + IPAddress: "10.0.1.10", + PrefixLength: 32, + }, + NICType: cns.InfraNIC, + }, + { + PodIPConfig: cns.IPSubnet{ + IPAddress: "2001:0db8:abcd:0015::0", + PrefixLength: 64, + }, + NICType: cns.InfraNIC, + }, + } + for i := range podIPInfo { + ipInfo := &podIPInfo[i] + err := middleware.SetRoutes(ipInfo) + if err == nil { + t.Errorf("SetRoutes should fail due to env var not set") + } + } +} diff --git a/cns/restserver/ipam.go b/cns/restserver/ipam.go index 2b9542ac90..e3d413d9e3 100644 --- a/cns/restserver/ipam.go +++ b/cns/restserver/ipam.go @@ -80,7 +80,7 @@ func (service *HTTPRestService) requestIPConfigHandlerHelper(ctx context.Context if err != nil { defer func() { logger.Errorf("failed to get SWIFTv2 IP config : %v. Releasing default IP config...", err) - _, err := service.releaseIPConfigHandlerHelper(ctx, ipconfigsRequest) + _, err = service.releaseIPConfigHandlerHelper(ctx, ipconfigsRequest) if err != nil { logger.Errorf("failed to release default IP config %v", err) } @@ -99,9 +99,9 @@ func (service *HTTPRestService) requestIPConfigHandlerHelper(ctx context.Context ipInfo := &podIPInfo[i] err := service.SWIFTv2Middleware.SetRoutes(ipInfo) if err != nil { - defer func() { + defer func() { //nolint:gocritic logger.Errorf("failed to set routes for SWIFTv2 IP config : %v. Releasing default IP config...", err) - _, err := service.releaseIPConfigHandlerHelper(ctx, ipconfigsRequest) + _, err = service.releaseIPConfigHandlerHelper(ctx, ipconfigsRequest) if err != nil { logger.Errorf("failed to release default IP config %v", err) } diff --git a/cns/restserver/ipam_test.go b/cns/restserver/ipam_test.go index 9b17b79393..aac099e5a5 100644 --- a/cns/restserver/ipam_test.go +++ b/cns/restserver/ipam_test.go @@ -1533,9 +1533,14 @@ func TestIPAMFailToRequestPartialIPsInPool(t *testing.T) { } } -func TestIPAMGetSWIFTv2IP(t *testing.T) { +func TestIPAMGetSWIFTv2IPSuccess(t *testing.T) { svc := getTestService() - svc.AttachSWIFTv2Middleware(middlewares.NewMockSWIFTv2Middleware()) + middleware := middlewares.NewMockSWIFTv2Middleware() + svc.AttachSWIFTv2Middleware(middleware) + + middleware.SetEnvVar() + defer middleware.UnsetEnvVar() //nolint:errcheck // ignore error + middleware.SetMTPNCReady() ncStates := []ncState{ { @@ -1589,3 +1594,62 @@ func TestIPAMGetSWIFTv2IP(t *testing.T) { assert.Equal(t, cns.DelegatedVMNIC, podIPInfo[2].NICType) assert.False(t, podIPInfo[2].SkipDefaultRoutes) } + +func TestIPAMGetSWIFTv2IPFailure(t *testing.T) { + svc := getTestService() + middleware := middlewares.NewMockSWIFTv2Middleware() + svc.AttachSWIFTv2Middleware(middleware) + ncStates := []ncState{ + { + ncID: testNCID, + ips: []string{ + testIP1, + }, + }, + { + ncID: testNCIDv6, + ips: []string{ + testIP1v6, + }, + }, + } + // Add Available Pod IP to state + for i := range ncStates { + ipconfigs := make(map[string]cns.IPConfigurationStatus, 0) + state := NewPodState(ncStates[i].ips[0], ipIDs[i][0], ncStates[i].ncID, types.Available, 0) + ipconfigs[state.ID] = state + err := UpdatePodIPConfigState(t, svc, ipconfigs, ncStates[i].ncID) + if err != nil { + t.Fatalf("Expected to not fail adding IPs to state: %+v", err) + } + } + req := cns.IPConfigsRequest{ + PodInterfaceID: testPod1Info.InterfaceID(), + InfraContainerID: testPod1Info.InfraContainerID(), + } + b, _ := testPod1Info.OrchestratorContext() + req.OrchestratorContext = b + req.DesiredIPAddresses = make([]string, 2) + req.DesiredIPAddresses[0] = testIP1 + req.DesiredIPAddresses[1] = testIP1v6 + _, err := svc.requestIPConfigHandlerHelper(context.TODO(), req) + if err == nil { + t.Fatalf("Expected failing requesting IPs due to MTPNC not ready") + } + available := svc.GetAvailableIPConfigs() + if len(available) != 2 { + t.Fatal("Expected available ips to be 2 since we expect the IP to not be assigned") + } + + middleware.SetMTPNCReady() + + _, err = svc.requestIPConfigHandlerHelper(context.TODO(), req) + if err == nil { + t.Fatalf("Expected failing requesting IPs due to not able to set routes") + } + + available = svc.GetAvailableIPConfigs() + if len(available) != 2 { + t.Fatal("Expected available ips to be 2 since we expect the IP to not be assigned") + } +} From 7b3987a5d77738f92af7c858ee5ecd439748989c Mon Sep 17 00:00:00 2001 From: Quang Nguyen Date: Tue, 24 Oct 2023 18:05:49 -0400 Subject: [PATCH 68/85] fix: parsing MTPNC as CIDR instead --- cns/middlewares/swiftV2.go | 16 ++++++++++++++-- cns/restserver/ipam.go | 10 +++++----- 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/cns/middlewares/swiftV2.go b/cns/middlewares/swiftV2.go index 68524a74cb..0ad73fdd55 100644 --- a/cns/middlewares/swiftV2.go +++ b/cns/middlewares/swiftV2.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "net" + "net/netip" "strings" "github.com/Azure/azure-container-networking/cns" @@ -67,10 +68,21 @@ func (m *SWIFTv2Middleware) GetIPConfig(ctx context.Context, podInfo cns.PodInfo if mtpnc.Status.PrimaryIP == "" || mtpnc.Status.MacAddress == "" || mtpnc.Status.NCID == "" || mtpnc.Status.GatewayIP == "" { return cns.PodIpInfo{}, errMTPNCNotReady } + // Parse MTPNC primaryIP to get the IP address and prefix length + p, err := netip.ParsePrefix(mtpnc.Status.PrimaryIP) + if err != nil { + return cns.PodIpInfo{}, fmt.Errorf("failed to parse MTPNC primaryIP %s : %w", mtpnc.Status.PrimaryIP, err) + } + // Get the IP address and prefix length + ip := p.Addr() + prefixSize := p.Bits() + if prefixSize != prefixLength { + return cns.PodIpInfo{}, fmt.Errorf("invalid prefix length %d for MTPNC primaryIP %s, prefix length must be %d", prefixSize, mtpnc.Status.PrimaryIP, prefixLength) + } podIPInfo := cns.PodIpInfo{ PodIPConfig: cns.IPSubnet{ - IPAddress: mtpnc.Status.PrimaryIP, - PrefixLength: prefixLength, + IPAddress: ip.String(), + PrefixLength: uint8(prefixSize), }, MacAddress: mtpnc.Status.MacAddress, NICType: cns.DelegatedVMNIC, diff --git a/cns/restserver/ipam.go b/cns/restserver/ipam.go index e3d413d9e3..63991e7cdb 100644 --- a/cns/restserver/ipam.go +++ b/cns/restserver/ipam.go @@ -79,8 +79,8 @@ func (service *HTTPRestService) requestIPConfigHandlerHelper(ctx context.Context SWIFTv2PodIPInfo, err := service.SWIFTv2Middleware.GetIPConfig(ctx, podInfo) if err != nil { defer func() { - logger.Errorf("failed to get SWIFTv2 IP config : %v. Releasing default IP config...", err) - _, err = service.releaseIPConfigHandlerHelper(ctx, ipconfigsRequest) + logger.Errorf("failed to get SWIFTv2 IP config %v, releasing default IP config...", err) + _, err := service.releaseIPConfigHandlerHelper(ctx, ipconfigsRequest) if err != nil { logger.Errorf("failed to release default IP config %v", err) } @@ -99,9 +99,9 @@ func (service *HTTPRestService) requestIPConfigHandlerHelper(ctx context.Context ipInfo := &podIPInfo[i] err := service.SWIFTv2Middleware.SetRoutes(ipInfo) if err != nil { - defer func() { //nolint:gocritic - logger.Errorf("failed to set routes for SWIFTv2 IP config : %v. Releasing default IP config...", err) - _, err = service.releaseIPConfigHandlerHelper(ctx, ipconfigsRequest) + defer func() { + logger.Errorf("failed to get SWIFTv2 IP config %v, releasing default IP config...", err) + _, err := service.releaseIPConfigHandlerHelper(ctx, ipconfigsRequest) if err != nil { logger.Errorf("failed to release default IP config %v", err) } From 28b0bfab53137f80de8485c167b32f897ba3ff83 Mon Sep 17 00:00:00 2001 From: Quang Nguyen Date: Tue, 24 Oct 2023 18:34:48 -0400 Subject: [PATCH 69/85] fix: toggle skipDefaultRoutes for infraNic to true --- cns/middlewares/swiftV2.go | 1 + cns/middlewares/swiftV2_test.go | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/cns/middlewares/swiftV2.go b/cns/middlewares/swiftV2.go index 0ad73fdd55..15550366ab 100644 --- a/cns/middlewares/swiftV2.go +++ b/cns/middlewares/swiftV2.go @@ -158,6 +158,7 @@ func (m *SWIFTv2Middleware) SetRoutes(podIPInfo *cns.PodIpInfo) error { podIPInfo.Routes = append(podIPInfo.Routes, serviceCIDRv6Route) } } + podIPInfo.SkipDefaultRoutes = true default: return errInvalidSWIFTv2NICType } diff --git a/cns/middlewares/swiftV2_test.go b/cns/middlewares/swiftV2_test.go index 59eb5ebf6e..83f93b2dcc 100644 --- a/cns/middlewares/swiftV2_test.go +++ b/cns/middlewares/swiftV2_test.go @@ -178,6 +178,12 @@ func TestSetRoutesSuccess(t *testing.T) { ipInfo := &podIPInfo[i] err := middleware.SetRoutes(ipInfo) assert.Equal(t, err, nil) + if ipInfo.NICType == cns.InfraNIC { + assert.Equal(t, ipInfo.SkipDefaultRoutes, true) + } else { + assert.Equal(t, ipInfo.SkipDefaultRoutes, false) + } + } for i := range podIPInfo { assert.DeepEqual(t, podIPInfo[i].Routes, desiredPodIPInfo[i].Routes) From b307254b783ac39688f1f6b6e17410f2a2a6aea1 Mon Sep 17 00:00:00 2001 From: Quang Nguyen Date: Thu, 26 Oct 2023 13:12:10 -0400 Subject: [PATCH 70/85] fix: add route for node cidr in ipv4 podipconfig --- cns/configuration/env.go | 12 +++++++++ cns/middlewares/mock/mockSWIFTv2.go | 4 +++ cns/middlewares/swiftV2.go | 10 ++++++++ cns/middlewares/swiftV2_test.go | 6 +++++ cns/restserver/ipam.go | 38 ++++++++++++++--------------- 5 files changed, 50 insertions(+), 20 deletions(-) diff --git a/cns/configuration/env.go b/cns/configuration/env.go index 3d391ef579..7431c5432c 100644 --- a/cns/configuration/env.go +++ b/cns/configuration/env.go @@ -17,6 +17,7 @@ const ( LabelPodSwiftV2 = "kubernetes.azure.com/pod-network" EnvPodCIDRs = "POD_CIDRs" EnvServiceCIDRs = "SERVICE_CIDRs" + EnvNodeCIDR = "NODE_CIDR" ) // ErrNodeNameUnset indicates the the $EnvNodeName variable is unset in the environment. @@ -28,6 +29,9 @@ var ErrPodCIDRsUnset = errors.Errorf("must declare %s environment variable", Env // ErrServiceCIDRsUnset indicates the the $EnvServiceCIDRs variable is unset in the environment. var ErrServiceCIDRsUnset = errors.Errorf("must declare %s environment variable", EnvServiceCIDRs) +// ErrNodeCIDRUnset indicates the the $EnvNodeCIDR variable is unset in the environment. +var ErrNodeCIDRUnset = errors.Errorf("must declare %s environment variable", EnvNodeCIDR) + // NodeName checks the environment variables for the NODENAME and returns it or an error if unset. func NodeName() (string, error) { nodeName := os.Getenv(EnvNodeName) @@ -57,3 +61,11 @@ func ServiceCIDRs() (string, error) { } return serviceCIDRs, nil } + +func NodeCIDR() (string, error) { + nodeCIDR := os.Getenv(EnvNodeCIDR) + if nodeCIDR == "" { + return "", ErrNodeCIDRUnset + } + return nodeCIDR, nil +} diff --git a/cns/middlewares/mock/mockSWIFTv2.go b/cns/middlewares/mock/mockSWIFTv2.go index c2106e3d58..f8355e92e4 100644 --- a/cns/middlewares/mock/mockSWIFTv2.go +++ b/cns/middlewares/mock/mockSWIFTv2.go @@ -56,6 +56,7 @@ func (m *MockSWIFTv2Middleware) SetMTPNCReady() { func (m *MockSWIFTv2Middleware) SetEnvVar() { os.Setenv(configuration.EnvPodCIDRs, "10.0.1.10/24") os.Setenv(configuration.EnvServiceCIDRs, "10.0.2.10/24") + os.Setenv(configuration.EnvNodeCIDR, "10.0.3.10/24") } func (m *MockSWIFTv2Middleware) UnsetEnvVar() error { @@ -65,6 +66,9 @@ func (m *MockSWIFTv2Middleware) UnsetEnvVar() error { if err := os.Unsetenv(configuration.EnvServiceCIDRs); err != nil { return fmt.Errorf("failed to unset env var %s : %w", configuration.EnvServiceCIDRs, err) } + if err := os.Unsetenv(configuration.EnvNodeCIDR); err != nil { + return fmt.Errorf("failed to unset env var %s : %w", configuration.EnvNodeCIDR, err) + } return nil } diff --git a/cns/middlewares/swiftV2.go b/cns/middlewares/swiftV2.go index 15550366ab..fcd2ccda0f 100644 --- a/cns/middlewares/swiftV2.go +++ b/cns/middlewares/swiftV2.go @@ -139,6 +139,16 @@ func (m *SWIFTv2Middleware) SetRoutes(podIPInfo *cns.PodIpInfo) error { } podIPInfo.Routes = append(podIPInfo.Routes, serviceCIDRv4Route) } + nodeCIDR, err := configuration.NodeCIDR() + if err != nil { + return fmt.Errorf("failed to get nodeCIDR from env : %w", err) + } + // route for nodeCIDR traffic + nodeCIDRRoute := cns.Route{ + IPAddress: nodeCIDR, + GatewayIPAddress: overlayGatewayv4, + } + podIPInfo.Routes = append(podIPInfo.Routes, nodeCIDRRoute) } else { // routes for IPv6 podCIDR traffic diff --git a/cns/middlewares/swiftV2_test.go b/cns/middlewares/swiftV2_test.go index 83f93b2dcc..ba67bc1f5e 100644 --- a/cns/middlewares/swiftV2_test.go +++ b/cns/middlewares/swiftV2_test.go @@ -26,11 +26,13 @@ var ( func setEnvVar() { os.Setenv(configuration.EnvPodCIDRs, "10.0.1.10/24,16A0:0010:AB00:001E::2/32") os.Setenv(configuration.EnvServiceCIDRs, "10.0.0.0/16,16A0:0010:AB00:0000::/32") + os.Setenv(configuration.EnvNodeCIDR, "10.240.0.1/16") } func unsetEnvVar() { os.Unsetenv(configuration.EnvPodCIDRs) os.Unsetenv(configuration.EnvServiceCIDRs) + os.Unsetenv(configuration.EnvNodeCIDR) } func TestValidateMultitenantIPConfigsRequestSuccess(t *testing.T) { @@ -141,6 +143,10 @@ func TestSetRoutesSuccess(t *testing.T) { IPAddress: "10.0.0.0/16", GatewayIPAddress: overlayGatewayv4, }, + { + IPAddress: "10.240.0.1/16", + GatewayIPAddress: overlayGatewayv4, + }, }, }, { diff --git a/cns/restserver/ipam.go b/cns/restserver/ipam.go index 63991e7cdb..48a69da345 100644 --- a/cns/restserver/ipam.go +++ b/cns/restserver/ipam.go @@ -37,6 +37,22 @@ func (service *HTTPRestService) requestIPConfigHandlerHelper(ctx context.Context }, errors.New("failed to validate ip config request") } + var SWIFTv2PodIPInfo cns.PodIpInfo + // Check if request is for pod with secondary interface(s) + if podInfo.SecondaryInterfacesExist() { + podIPInfo, err := service.SWIFTv2Middleware.GetIPConfig(ctx, podInfo) + if err != nil { + return &cns.IPConfigsResponse{ + Response: cns.Response{ + ReturnCode: types.FailedToAllocateIPConfig, + Message: fmt.Sprintf("AllocateIPConfig failed: %v, IP config request is %v", err, ipconfigsRequest), + }, + PodIPInfo: []cns.PodIpInfo{}, + }, errors.Wrapf(err, "failed to get SWIFTv2 IP config %v", ipconfigsRequest) + } + SWIFTv2PodIPInfo = podIPInfo + } + // record a pod requesting an IP service.podsPendingIPAssignment.Push(podInfo.Key()) @@ -73,26 +89,8 @@ func (service *HTTPRestService) requestIPConfigHandlerHelper(ctx context.Context } } - // Check if request is for pod with secondary interface(s) - if podInfo.SecondaryInterfacesExist() { - // In the future, if we have multiple scenario with secondary interfaces, we can add a switch case here - SWIFTv2PodIPInfo, err := service.SWIFTv2Middleware.GetIPConfig(ctx, podInfo) - if err != nil { - defer func() { - logger.Errorf("failed to get SWIFTv2 IP config %v, releasing default IP config...", err) - _, err := service.releaseIPConfigHandlerHelper(ctx, ipconfigsRequest) - if err != nil { - logger.Errorf("failed to release default IP config %v", err) - } - }() - return &cns.IPConfigsResponse{ - Response: cns.Response{ - ReturnCode: types.FailedToAllocateIPConfig, - Message: fmt.Sprintf("AllocateIPConfig failed: %v, IP config request is %v", err, ipconfigsRequest), - }, - PodIPInfo: []cns.PodIpInfo{}, - }, errors.Wrapf(err, "failed to get SWIFTv2 IP config %v", ipconfigsRequest) - } + // Check if SWIFTv2 pod IP info is set + if SWIFTv2PodIPInfo.PodIPConfig.IPAddress != "" { podIPInfo = append(podIPInfo, SWIFTv2PodIPInfo) // Setting up routes for SWIFTv2 scenario for i := range podIPInfo { From 38fc47187268a407703d055b587609cd515fa4a6 Mon Sep 17 00:00:00 2001 From: Quang Nguyen Date: Thu, 26 Oct 2023 16:26:05 -0400 Subject: [PATCH 71/85] feat: add node cidrs route --- cns/configuration/env.go | 16 +++++------ cns/middlewares/mock/mockClient.go | 2 +- cns/middlewares/mock/mockSWIFTv2.go | 6 ++-- cns/middlewares/swiftV2.go | 44 ++++++++++++++++++++--------- cns/middlewares/swiftV2_test.go | 8 ++++-- cns/restserver/ipam.go | 16 +++-------- cns/restserver/ipam_test.go | 5 ---- 7 files changed, 53 insertions(+), 44 deletions(-) diff --git a/cns/configuration/env.go b/cns/configuration/env.go index 7431c5432c..6b16668cb2 100644 --- a/cns/configuration/env.go +++ b/cns/configuration/env.go @@ -17,7 +17,7 @@ const ( LabelPodSwiftV2 = "kubernetes.azure.com/pod-network" EnvPodCIDRs = "POD_CIDRs" EnvServiceCIDRs = "SERVICE_CIDRs" - EnvNodeCIDR = "NODE_CIDR" + EnvNodeCIDRs = "NODE_CIDRs" ) // ErrNodeNameUnset indicates the the $EnvNodeName variable is unset in the environment. @@ -29,8 +29,8 @@ var ErrPodCIDRsUnset = errors.Errorf("must declare %s environment variable", Env // ErrServiceCIDRsUnset indicates the the $EnvServiceCIDRs variable is unset in the environment. var ErrServiceCIDRsUnset = errors.Errorf("must declare %s environment variable", EnvServiceCIDRs) -// ErrNodeCIDRUnset indicates the the $EnvNodeCIDR variable is unset in the environment. -var ErrNodeCIDRUnset = errors.Errorf("must declare %s environment variable", EnvNodeCIDR) +// ErrNodeCIDRsUnset indicates the the $EnvNodeCIDRs variable is unset in the environment. +var ErrNodeCIDRsUnset = errors.Errorf("must declare %s environment variable", EnvNodeCIDRs) // NodeName checks the environment variables for the NODENAME and returns it or an error if unset. func NodeName() (string, error) { @@ -62,10 +62,10 @@ func ServiceCIDRs() (string, error) { return serviceCIDRs, nil } -func NodeCIDR() (string, error) { - nodeCIDR := os.Getenv(EnvNodeCIDR) - if nodeCIDR == "" { - return "", ErrNodeCIDRUnset +func NodeCIDRs() (string, error) { + nodeCIDRs := os.Getenv(EnvNodeCIDRs) + if nodeCIDRs == "" { + return "", ErrNodeCIDRsUnset } - return nodeCIDR, nil + return nodeCIDRs, nil } diff --git a/cns/middlewares/mock/mockClient.go b/cns/middlewares/mock/mockClient.go index 6432d55852..b888c735fe 100644 --- a/cns/middlewares/mock/mockClient.go +++ b/cns/middlewares/mock/mockClient.go @@ -29,7 +29,7 @@ func NewMockClient() *MockClient { testPod1.Labels[configuration.LabelPodSwiftV2] = "true" testMTPNC1 := v1alpha1.MultitenantPodNetworkConfig{} - testMTPNC1.Status.PrimaryIP = "192.168.0.1" + testMTPNC1.Status.PrimaryIP = "192.168.0.1/32" testMTPNC1.Status.MacAddress = "00:00:00:00:00:00" testMTPNC1.Status.GatewayIP = "10.0.0.1" testMTPNC1.Status.NCID = "testncid" diff --git a/cns/middlewares/mock/mockSWIFTv2.go b/cns/middlewares/mock/mockSWIFTv2.go index f8355e92e4..9602521773 100644 --- a/cns/middlewares/mock/mockSWIFTv2.go +++ b/cns/middlewares/mock/mockSWIFTv2.go @@ -56,7 +56,7 @@ func (m *MockSWIFTv2Middleware) SetMTPNCReady() { func (m *MockSWIFTv2Middleware) SetEnvVar() { os.Setenv(configuration.EnvPodCIDRs, "10.0.1.10/24") os.Setenv(configuration.EnvServiceCIDRs, "10.0.2.10/24") - os.Setenv(configuration.EnvNodeCIDR, "10.0.3.10/24") + os.Setenv(configuration.EnvNodeCIDRs, "10.0.3.10/24") } func (m *MockSWIFTv2Middleware) UnsetEnvVar() error { @@ -66,8 +66,8 @@ func (m *MockSWIFTv2Middleware) UnsetEnvVar() error { if err := os.Unsetenv(configuration.EnvServiceCIDRs); err != nil { return fmt.Errorf("failed to unset env var %s : %w", configuration.EnvServiceCIDRs, err) } - if err := os.Unsetenv(configuration.EnvNodeCIDR); err != nil { - return fmt.Errorf("failed to unset env var %s : %w", configuration.EnvNodeCIDR, err) + if err := os.Unsetenv(configuration.EnvNodeCIDRs); err != nil { + return fmt.Errorf("failed to unset env var %s : %w", configuration.EnvNodeCIDRs, err) } return nil } diff --git a/cns/middlewares/swiftV2.go b/cns/middlewares/swiftV2.go index fcd2ccda0f..0753fe3b6e 100644 --- a/cns/middlewares/swiftV2.go +++ b/cns/middlewares/swiftV2.go @@ -18,8 +18,9 @@ import ( ) var ( - errMTPNCNotReady = errors.New("mtpnc is not ready") - errInvalidSWIFTv2NICType = errors.New("invalid NIC type for SWIFT v2 scenario") + errMTPNCNotReady = errors.New("mtpnc is not ready") + errInvalidSWIFTv2NICType = errors.New("invalid NIC type for SWIFT v2 scenario") + errInvalidMTPNCPrefixLength = errors.New("invalid prefix length for MTPNC primaryIP, must be 32") ) const ( @@ -77,7 +78,7 @@ func (m *SWIFTv2Middleware) GetIPConfig(ctx context.Context, podInfo cns.PodInfo ip := p.Addr() prefixSize := p.Bits() if prefixSize != prefixLength { - return cns.PodIpInfo{}, fmt.Errorf("invalid prefix length %d for MTPNC primaryIP %s, prefix length must be %d", prefixSize, mtpnc.Status.PrimaryIP, prefixLength) + return cns.PodIpInfo{}, fmt.Errorf("%w, MTPNC primaryIP prefix length is %d", errInvalidMTPNCPrefixLength, prefixSize) } podIPInfo := cns.PodIpInfo{ PodIPConfig: cns.IPSubnet{ @@ -104,6 +105,17 @@ func (m *SWIFTv2Middleware) SetRoutes(podIPInfo *cns.PodIpInfo) error { } podIPInfo.Routes = []cns.Route{route} case cns.InfraNIC: + // Get and parse nodeCIDRs from env + nodeCIDRs, err := configuration.NodeCIDRs() + if err != nil { + return fmt.Errorf("failed to get nodeCIDR from env : %w", err) + } + nodeCIDRsv4, nodeCIDRsv6, err := parseCIDRs(nodeCIDRs) + if err != nil { + return fmt.Errorf("failed to parse nodeCIDRs : %w", err) + } + + // Get and parse podCIDRs from env podCIDRs, err := configuration.PodCIDRs() if err != nil { return fmt.Errorf("failed to get podCIDRs from env : %w", err) @@ -113,6 +125,7 @@ func (m *SWIFTv2Middleware) SetRoutes(podIPInfo *cns.PodIpInfo) error { return fmt.Errorf("failed to parse podCIDRs : %w", err) } + // Get and parse serviceCIDRs from env serviceCIDRs, err := configuration.ServiceCIDRs() if err != nil { return fmt.Errorf("failed to get serviceCIDRs from env : %w", err) @@ -139,17 +152,14 @@ func (m *SWIFTv2Middleware) SetRoutes(podIPInfo *cns.PodIpInfo) error { } podIPInfo.Routes = append(podIPInfo.Routes, serviceCIDRv4Route) } - nodeCIDR, err := configuration.NodeCIDR() - if err != nil { - return fmt.Errorf("failed to get nodeCIDR from env : %w", err) - } - // route for nodeCIDR traffic - nodeCIDRRoute := cns.Route{ - IPAddress: nodeCIDR, - GatewayIPAddress: overlayGatewayv4, + // route for IPv4 nodeCIDR traffic + for _, nodeCIDRv4 := range nodeCIDRsv4 { + nodeCIDRv4Route := cns.Route{ + IPAddress: nodeCIDRv4, + GatewayIPAddress: overlayGatewayv4, + } + podIPInfo.Routes = append(podIPInfo.Routes, nodeCIDRv4Route) } - podIPInfo.Routes = append(podIPInfo.Routes, nodeCIDRRoute) - } else { // routes for IPv6 podCIDR traffic for _, podCIDRv6 := range podCIDRv6 { @@ -167,6 +177,14 @@ func (m *SWIFTv2Middleware) SetRoutes(podIPInfo *cns.PodIpInfo) error { } podIPInfo.Routes = append(podIPInfo.Routes, serviceCIDRv6Route) } + // route for IPv6 nodeCIDR traffic + for _, nodeCIDRv6 := range nodeCIDRsv6 { + nodeCIDRv6Route := cns.Route{ + IPAddress: nodeCIDRv6, + GatewayIPAddress: overlayGatewayV6, + } + podIPInfo.Routes = append(podIPInfo.Routes, nodeCIDRv6Route) + } } podIPInfo.SkipDefaultRoutes = true default: diff --git a/cns/middlewares/swiftV2_test.go b/cns/middlewares/swiftV2_test.go index ba67bc1f5e..60b41221c6 100644 --- a/cns/middlewares/swiftV2_test.go +++ b/cns/middlewares/swiftV2_test.go @@ -26,13 +26,13 @@ var ( func setEnvVar() { os.Setenv(configuration.EnvPodCIDRs, "10.0.1.10/24,16A0:0010:AB00:001E::2/32") os.Setenv(configuration.EnvServiceCIDRs, "10.0.0.0/16,16A0:0010:AB00:0000::/32") - os.Setenv(configuration.EnvNodeCIDR, "10.240.0.1/16") + os.Setenv(configuration.EnvNodeCIDRs, "10.240.0.1/16,16A0:0020:AB00:0000::/32") } func unsetEnvVar() { os.Unsetenv(configuration.EnvPodCIDRs) os.Unsetenv(configuration.EnvServiceCIDRs) - os.Unsetenv(configuration.EnvNodeCIDR) + os.Unsetenv(configuration.EnvNodeCIDRs) } func TestValidateMultitenantIPConfigsRequestSuccess(t *testing.T) { @@ -164,6 +164,10 @@ func TestSetRoutesSuccess(t *testing.T) { IPAddress: "16A0:0010:AB00:0000::/32", GatewayIPAddress: overlayGatewayV6, }, + { + IPAddress: "16A0:0020:AB00:0000::/32", + GatewayIPAddress: overlayGatewayV6, + }, }, }, { diff --git a/cns/restserver/ipam.go b/cns/restserver/ipam.go index 48a69da345..218106dcbf 100644 --- a/cns/restserver/ipam.go +++ b/cns/restserver/ipam.go @@ -95,22 +95,14 @@ func (service *HTTPRestService) requestIPConfigHandlerHelper(ctx context.Context // Setting up routes for SWIFTv2 scenario for i := range podIPInfo { ipInfo := &podIPInfo[i] - err := service.SWIFTv2Middleware.SetRoutes(ipInfo) - if err != nil { - defer func() { - logger.Errorf("failed to get SWIFTv2 IP config %v, releasing default IP config...", err) - _, err := service.releaseIPConfigHandlerHelper(ctx, ipconfigsRequest) - if err != nil { - logger.Errorf("failed to release default IP config %v", err) - } - }() + if err := service.SWIFTv2Middleware.SetRoutes(ipInfo); err != nil { return &cns.IPConfigsResponse{ Response: cns.Response{ - ReturnCode: types.FailedToAllocateIPConfig, - Message: fmt.Sprintf("AllocateIPConfig failed: %v, IP config request is %v", err, ipconfigsRequest), + ReturnCode: types.UnexpectedError, + Message: fmt.Sprintf("failed to set routes for SWIFT v2: %v ", err), }, PodIPInfo: []cns.PodIpInfo{}, - }, errors.Wrapf(err, "failed to set SWIFTv2 routes %v", ipconfigsRequest) + }, fmt.Errorf("failed to set routes for SWIFT v2: %w", err) } } } diff --git a/cns/restserver/ipam_test.go b/cns/restserver/ipam_test.go index aac099e5a5..5c5dc95bd5 100644 --- a/cns/restserver/ipam_test.go +++ b/cns/restserver/ipam_test.go @@ -1647,9 +1647,4 @@ func TestIPAMGetSWIFTv2IPFailure(t *testing.T) { if err == nil { t.Fatalf("Expected failing requesting IPs due to not able to set routes") } - - available = svc.GetAvailableIPConfigs() - if len(available) != 2 { - t.Fatal("Expected available ips to be 2 since we expect the IP to not be assigned") - } } From d1522cd36288ac8f4abf615aba75ab3c67ebfaa4 Mon Sep 17 00:00:00 2001 From: Quang Nguyen Date: Thu, 26 Oct 2023 16:31:39 -0400 Subject: [PATCH 72/85] fix: linter --- cns/restserver/ipam.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/cns/restserver/ipam.go b/cns/restserver/ipam.go index 218106dcbf..2973260452 100644 --- a/cns/restserver/ipam.go +++ b/cns/restserver/ipam.go @@ -37,9 +37,13 @@ func (service *HTTPRestService) requestIPConfigHandlerHelper(ctx context.Context }, errors.New("failed to validate ip config request") } + // record a pod requesting an IP + service.podsPendingIPAssignment.Push(podInfo.Key()) + var SWIFTv2PodIPInfo cns.PodIpInfo // Check if request is for pod with secondary interface(s) if podInfo.SecondaryInterfacesExist() { + // In the future, if we have multiple scenario with secondary interfaces, we can add a switch case here podIPInfo, err := service.SWIFTv2Middleware.GetIPConfig(ctx, podInfo) if err != nil { return &cns.IPConfigsResponse{ @@ -53,9 +57,7 @@ func (service *HTTPRestService) requestIPConfigHandlerHelper(ctx context.Context SWIFTv2PodIPInfo = podIPInfo } - // record a pod requesting an IP - service.podsPendingIPAssignment.Push(podInfo.Key()) - + // Assign default IP config podIPInfo, err := requestIPConfigsHelper(service, ipconfigsRequest) if err != nil { return &cns.IPConfigsResponse{ From de1c82d154a6d19028b27608ef2f32b669167ec5 Mon Sep 17 00:00:00 2001 From: Quang Nguyen Date: Thu, 26 Oct 2023 16:40:19 -0400 Subject: [PATCH 73/85] address comments --- cns/restserver/ipam.go | 55 +++++++++++++++++++++---------------- cns/restserver/ipam_test.go | 5 ++++ 2 files changed, 36 insertions(+), 24 deletions(-) diff --git a/cns/restserver/ipam.go b/cns/restserver/ipam.go index 2973260452..5bd0101a3e 100644 --- a/cns/restserver/ipam.go +++ b/cns/restserver/ipam.go @@ -24,7 +24,6 @@ var ( ErrNoNCs = errors.New("No NCs found in the CNS internal state") ) -// requestIPConfigHandlerHelper validates the request, assigns IPs, and returns a response func (service *HTTPRestService) requestIPConfigHandlerHelper(ctx context.Context, ipconfigsRequest cns.IPConfigsRequest) (*cns.IPConfigsResponse, error) { // For SWIFT v2 scenario, the validator function will also modify the ipconfigsRequest. podInfo, returnCode, returnMessage := service.validateIPConfigsRequest(ctx, ipconfigsRequest) @@ -40,24 +39,6 @@ func (service *HTTPRestService) requestIPConfigHandlerHelper(ctx context.Context // record a pod requesting an IP service.podsPendingIPAssignment.Push(podInfo.Key()) - var SWIFTv2PodIPInfo cns.PodIpInfo - // Check if request is for pod with secondary interface(s) - if podInfo.SecondaryInterfacesExist() { - // In the future, if we have multiple scenario with secondary interfaces, we can add a switch case here - podIPInfo, err := service.SWIFTv2Middleware.GetIPConfig(ctx, podInfo) - if err != nil { - return &cns.IPConfigsResponse{ - Response: cns.Response{ - ReturnCode: types.FailedToAllocateIPConfig, - Message: fmt.Sprintf("AllocateIPConfig failed: %v, IP config request is %v", err, ipconfigsRequest), - }, - PodIPInfo: []cns.PodIpInfo{}, - }, errors.Wrapf(err, "failed to get SWIFTv2 IP config %v", ipconfigsRequest) - } - SWIFTv2PodIPInfo = podIPInfo - } - - // Assign default IP config podIPInfo, err := requestIPConfigsHelper(service, ipconfigsRequest) if err != nil { return &cns.IPConfigsResponse{ @@ -91,20 +72,46 @@ func (service *HTTPRestService) requestIPConfigHandlerHelper(ctx context.Context } } - // Check if SWIFTv2 pod IP info is set - if SWIFTv2PodIPInfo.PodIPConfig.IPAddress != "" { + // Check if request is for pod with secondary interface(s) + if podInfo.SecondaryInterfacesExist() { + // In the future, if we have multiple scenario with secondary interfaces, we can add a switch case here + SWIFTv2PodIPInfo, err := service.SWIFTv2Middleware.GetIPConfig(ctx, podInfo) + if err != nil { + defer func() { + logger.Errorf("failed to get SWIFTv2 IP config : %v. Releasing default IP config...", err) + _, err = service.releaseIPConfigHandlerHelper(ctx, ipconfigsRequest) + if err != nil { + logger.Errorf("failed to release default IP config %v", err) + } + }() + return &cns.IPConfigsResponse{ + Response: cns.Response{ + ReturnCode: types.FailedToAllocateIPConfig, + Message: fmt.Sprintf("AllocateIPConfig failed: %v, IP config request is %v", err, ipconfigsRequest), + }, + PodIPInfo: []cns.PodIpInfo{}, + }, errors.Wrapf(err, "failed to get SWIFTv2 IP config %v", ipconfigsRequest) + } podIPInfo = append(podIPInfo, SWIFTv2PodIPInfo) // Setting up routes for SWIFTv2 scenario for i := range podIPInfo { ipInfo := &podIPInfo[i] - if err := service.SWIFTv2Middleware.SetRoutes(ipInfo); err != nil { + err := service.SWIFTv2Middleware.SetRoutes(ipInfo) + if err != nil { + defer func() { //nolint:gocritic + logger.Errorf("failed to set routes for SWIFTv2 IP config : %v. Releasing default IP config...", err) + _, err = service.releaseIPConfigHandlerHelper(ctx, ipconfigsRequest) + if err != nil { + logger.Errorf("failed to release default IP config %v", err) + } + }() return &cns.IPConfigsResponse{ Response: cns.Response{ ReturnCode: types.UnexpectedError, - Message: fmt.Sprintf("failed to set routes for SWIFT v2: %v ", err), + Message: fmt.Sprintf("failed to set SWIFTv2 routes %v", err), }, PodIPInfo: []cns.PodIpInfo{}, - }, fmt.Errorf("failed to set routes for SWIFT v2: %w", err) + }, errors.Wrapf(err, "failed to set SWIFTv2 routes %v", ipconfigsRequest) } } } diff --git a/cns/restserver/ipam_test.go b/cns/restserver/ipam_test.go index 5c5dc95bd5..aac099e5a5 100644 --- a/cns/restserver/ipam_test.go +++ b/cns/restserver/ipam_test.go @@ -1647,4 +1647,9 @@ func TestIPAMGetSWIFTv2IPFailure(t *testing.T) { if err == nil { t.Fatalf("Expected failing requesting IPs due to not able to set routes") } + + available = svc.GetAvailableIPConfigs() + if len(available) != 2 { + t.Fatal("Expected available ips to be 2 since we expect the IP to not be assigned") + } } From 5ef96232add92f83ac1ce4bdd4a8114abaa4b2fb Mon Sep 17 00:00:00 2001 From: Quang Nguyen Date: Thu, 26 Oct 2023 20:53:58 -0400 Subject: [PATCH 74/85] fix: minor logs formatting --- cns/restserver/ipam.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cns/restserver/ipam.go b/cns/restserver/ipam.go index 5bd0101a3e..5bb9dffda1 100644 --- a/cns/restserver/ipam.go +++ b/cns/restserver/ipam.go @@ -81,7 +81,7 @@ func (service *HTTPRestService) requestIPConfigHandlerHelper(ctx context.Context logger.Errorf("failed to get SWIFTv2 IP config : %v. Releasing default IP config...", err) _, err = service.releaseIPConfigHandlerHelper(ctx, ipconfigsRequest) if err != nil { - logger.Errorf("failed to release default IP config %v", err) + logger.Errorf("failed to release default IP config : %v", err) } }() return &cns.IPConfigsResponse{ @@ -90,7 +90,7 @@ func (service *HTTPRestService) requestIPConfigHandlerHelper(ctx context.Context Message: fmt.Sprintf("AllocateIPConfig failed: %v, IP config request is %v", err, ipconfigsRequest), }, PodIPInfo: []cns.PodIpInfo{}, - }, errors.Wrapf(err, "failed to get SWIFTv2 IP config %v", ipconfigsRequest) + }, errors.Wrapf(err, "failed to get SWIFTv2 IP config : %v", ipconfigsRequest) } podIPInfo = append(podIPInfo, SWIFTv2PodIPInfo) // Setting up routes for SWIFTv2 scenario @@ -102,16 +102,16 @@ func (service *HTTPRestService) requestIPConfigHandlerHelper(ctx context.Context logger.Errorf("failed to set routes for SWIFTv2 IP config : %v. Releasing default IP config...", err) _, err = service.releaseIPConfigHandlerHelper(ctx, ipconfigsRequest) if err != nil { - logger.Errorf("failed to release default IP config %v", err) + logger.Errorf("failed to release default IP config : %v", err) } }() return &cns.IPConfigsResponse{ Response: cns.Response{ ReturnCode: types.UnexpectedError, - Message: fmt.Sprintf("failed to set SWIFTv2 routes %v", err), + Message: fmt.Sprintf("failed to set SWIFTv2 routes : %v", err), }, PodIPInfo: []cns.PodIpInfo{}, - }, errors.Wrapf(err, "failed to set SWIFTv2 routes %v", ipconfigsRequest) + }, errors.Wrapf(err, "failed to set SWIFTv2 routes : %v", ipconfigsRequest) } } } From adb18139356976e04a8c7f9c2ca7fc59a63d22c0 Mon Sep 17 00:00:00 2001 From: Quang Nguyen Date: Fri, 27 Oct 2023 11:49:45 -0400 Subject: [PATCH 75/85] feat: move cns yaml for swiftv2 scenario to a diff file + more logging for swiftv2middleware --- cns/azure-cns-swiftv2.yaml | 228 +++++++++++++++++++++++++++++++++++++ cns/azure-cns.yaml | 104 ++++------------- cns/middlewares/swiftV2.go | 5 + 3 files changed, 256 insertions(+), 81 deletions(-) create mode 100644 cns/azure-cns-swiftv2.yaml diff --git a/cns/azure-cns-swiftv2.yaml b/cns/azure-cns-swiftv2.yaml new file mode 100644 index 0000000000..0cd7700aab --- /dev/null +++ b/cns/azure-cns-swiftv2.yaml @@ -0,0 +1,228 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: azure-cns + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: acn-multitenancy-editor +labels: + addonmanager.kubernetes.io/mode: Reconcile +rules: + - apiGroups: ["acn.azure.com"] + resources: ["clustersubnetstates"] + verbs: ["get", "list", "watch"] + - apiGroups: ["multitenancy.acn.azure.com"] + resources: + ["multitenantpodnetworkconfigs", "podnetworkinstances", "podnetworks"] + verbs: ["get", "list", "watch", "patch", "update"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: acn-multitenancy-editor-binding +subjects: + - kind: ServiceAccount + name: azure-cns + namespace: kube-system +roleRef: + kind: ClusterRole + name: acn-multitenancy-editor + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + namespace: kube-system + name: nodeNetConfigEditor +rules: + - apiGroups: ["acn.azure.com"] + resources: ["nodenetworkconfigs"] + verbs: ["get", "list", "watch", "patch", "update"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: pod-reader-all-namespaces +rules: + - apiGroups: [""] + resources: ["pods"] + verbs: ["get", "watch", "list"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: nodeNetConfigEditorRoleBinding + namespace: kube-system +subjects: + - kind: ServiceAccount + name: azure-cns + namespace: kube-system +roleRef: + kind: Role + name: nodeNetConfigEditor + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: pod-reader-all-namespaces-binding +subjects: + - kind: ServiceAccount + name: azure-cns + namespace: kube-system +roleRef: + kind: ClusterRole + name: pod-reader-all-namespaces + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: azure-cns + namespace: kube-system + labels: + app: azure-cns +spec: + selector: + matchLabels: + k8s-app: azure-cns + template: + metadata: + labels: + k8s-app: azure-cns + annotations: + cluster-autoscaler.kubernetes.io/daemonset-pod: "true" + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.azure.com/cluster + operator: Exists + - key: type + operator: NotIn + values: + - virtual-kubelet + - key: beta.kubernetes.io/os + operator: In + values: + - linux + priorityClassName: system-node-critical + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - operator: "Exists" + effect: NoExecute + - operator: "Exists" + effect: NoSchedule + containers: + - name: cns-container + image: mcr.microsoft.com/containernetworking/azure-cns:v1.4.7 + imagePullPolicy: IfNotPresent + args: + ["-c", "tcp://$(CNSIpAddress):$(CNSPort)", "-t", "$(CNSLogTarget)"] + volumeMounts: + - name: log + mountPath: /var/log + - name: cns-state + mountPath: /var/lib/azure-network + - name: azure-endpoints + mountPath: /var/run/azure-cns/ + - name: cns-config + mountPath: /etc/azure-cns + - name: cni-bin + mountPath: /opt/cni/bin + - name: azure-vnet + mountPath: /var/run/azure-vnet + - name: legacy-cni-state + mountPath: /var/run/azure-vnet.json + ports: + - containerPort: 10090 + env: + - name: CNSIpAddress + value: "127.0.0.1" + - name: CNSPort + value: "10090" + - name: CNSLogTarget + value: "stdoutfile" + - name: CNS_CONFIGURATION_PATH + value: /etc/azure-cns/cns_config.json + - name: NODENAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: POD_CIDRs + value: 10.0.0.0/24 # example value + - name: SERVICE_CIDRs + value: 10.0.0.0/25 # example value + - name: NODE_CIDRs + value: 198.19.0.128/25 # example value + hostNetwork: true + volumes: + - name: azure-endpoints + hostPath: + path: /var/run/azure-cns/ + type: DirectoryOrCreate + - name: log + hostPath: + path: /var/log + type: Directory + - name: cns-state + hostPath: + path: /var/lib/azure-network + type: DirectoryOrCreate + - name: cni-bin + hostPath: + path: /opt/cni/bin + type: Directory + - name: azure-vnet + hostPath: + path: /var/run/azure-vnet + type: DirectoryOrCreate + - name: legacy-cni-state + hostPath: + path: /var/run/azure-vnet.json + type: FileOrCreate + - name: cns-config + configMap: + name: cns-config + serviceAccountName: azure-cns +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: cns-config + namespace: kube-system +data: + cns_config.json: | + { + "TelemetrySettings": { + "TelemetryBatchSizeBytes": 16384, + "TelemetryBatchIntervalInSecs": 15, + "RefreshIntervalInSecs": 15, + "DisableAll": false, + "HeartBeatIntervalInMins": 30, + "DebugMode": false, + "SnapshotIntervalInMins": 60 + }, + "ManagedSettings": { + "PrivateEndpoint": "", + "InfrastructureNetworkID": "", + "NodeID": "", + "NodeSyncIntervalInSeconds": 30 + }, + "ChannelMode": "CRD", + "InitializeFromCNI": true, + "ManageEndpointState": false, + "ProgramSNATIPTables" : false + } + +# Toggle ManageEndpointState and ProgramSNATIPTables to true for delegated IPAM use case. diff --git a/cns/azure-cns.yaml b/cns/azure-cns.yaml index 771fff2989..7975a0a9f6 100644 --- a/cns/azure-cns.yaml +++ b/cns/azure-cns.yaml @@ -1,68 +1,12 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: azure-cns - namespace: kube-system ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: acn-multitenancy-editor -labels: - addonmanager.kubernetes.io/mode: Reconcile -rules: - - apiGroups: ["acn.azure.com"] - resources: ["clustersubnetstates"] - verbs: ["get", "list", "watch"] - - apiGroups: ["multitenancy.acn.azure.com"] - resources: - ["multitenantpodnetworkconfigs", "podnetworkinstances", "podnetworks"] - verbs: ["get", "list", "watch", "patch", "update"] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: acn-multitenancy-editor-binding -subjects: - - kind: ServiceAccount - name: azure-cns - namespace: kube-system -roleRef: - kind: ClusterRole - name: acn-multitenancy-editor - apiGroup: rbac.authorization.k8s.io ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - namespace: kube-system - name: nodeNetConfigEditor -rules: - - apiGroups: ["acn.azure.com"] - resources: ["nodenetworkconfigs"] - verbs: ["get", "list", "watch", "patch", "update"] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: pod-reader-all-namespaces -rules: - - apiGroups: [""] - resources: ["pods"] - verbs: ["get", "watch", "list"] - - apiGroups: [""] - resources: ["nodes"] - verbs: ["get"] ---- -apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding metadata: name: nodeNetConfigEditorRoleBinding namespace: kube-system subjects: - - kind: ServiceAccount - name: azure-cns - namespace: kube-system +- kind: ServiceAccount + name: azure-cns + namespace: kube-system roleRef: kind: Role name: nodeNetConfigEditor @@ -73,9 +17,9 @@ kind: ClusterRoleBinding metadata: name: pod-reader-all-namespaces-binding subjects: - - kind: ServiceAccount - name: azure-cns - namespace: kube-system +- kind: ServiceAccount + name: azure-cns + namespace: kube-system roleRef: kind: ClusterRole name: pod-reader-all-namespaces @@ -103,17 +47,17 @@ spec: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - - matchExpressions: - - key: kubernetes.azure.com/cluster - operator: Exists - - key: type - operator: NotIn - values: - - virtual-kubelet - - key: beta.kubernetes.io/os - operator: In - values: - - linux + - matchExpressions: + - key: kubernetes.azure.com/cluster + operator: Exists + - key: type + operator: NotIn + values: + - virtual-kubelet + - key: beta.kubernetes.io/os + operator: In + values: + - linux priorityClassName: system-node-critical tolerations: - key: CriticalAddonsOnly @@ -126,8 +70,7 @@ spec: - name: cns-container image: mcr.microsoft.com/containernetworking/azure-cns:v1.4.7 imagePullPolicy: IfNotPresent - args: - ["-c", "tcp://$(CNSIpAddress):$(CNSPort)", "-t", "$(CNSLogTarget)"] + args: [ "-c", "tcp://$(CNSIpAddress):$(CNSPort)", "-t", "$(CNSLogTarget)"] volumeMounts: - name: log mountPath: /var/log @@ -156,9 +99,9 @@ spec: value: /etc/azure-cns/cns_config.json - name: NODENAME valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName hostNetwork: true volumes: - name: azure-endpoints @@ -218,5 +161,4 @@ data: "ManageEndpointState": false, "ProgramSNATIPTables" : false } - -# Toggle ManageEndpointState and ProgramSNATIPTables to true for delegated IPAM use case. +# Toggle ManageEndpointState and ProgramSNATIPTables to true for delegated IPAM use case. \ No newline at end of file diff --git a/cns/middlewares/swiftV2.go b/cns/middlewares/swiftV2.go index 0753fe3b6e..20b3b47b6b 100644 --- a/cns/middlewares/swiftV2.go +++ b/cns/middlewares/swiftV2.go @@ -10,6 +10,7 @@ import ( "github.com/Azure/azure-container-networking/cns" "github.com/Azure/azure-container-networking/cns/configuration" + "github.com/Azure/azure-container-networking/cns/logger" "github.com/Azure/azure-container-networking/cns/types" "github.com/Azure/azure-container-networking/crd/multitenancy/api/v1alpha1" v1 "k8s.io/api/core/v1" @@ -38,6 +39,7 @@ type SWIFTv2Middleware struct { func (m *SWIFTv2Middleware) ValidateIPConfigsRequest(ctx context.Context, req *cns.IPConfigsRequest) (respCode types.ResponseCode, message string) { // Retrieve the pod from the cluster podInfo, err := cns.UnmarshalPodInfo(req.OrchestratorContext) + logger.Debugf("[SWIFTv2Middleware] validate ipconfigs request for pod %s", podInfo.Name()) if err != nil { errBuf := fmt.Sprintf("unmarshalling pod info from ipconfigs request %v failed with error %v", req, err) return types.UnexpectedError, errBuf @@ -53,6 +55,7 @@ func (m *SWIFTv2Middleware) ValidateIPConfigsRequest(ctx context.Context, req *c if _, ok := pod.Labels[configuration.LabelPodSwiftV2]; ok { req.SecondaryInterfacesExist = true } + logger.Debugf("[SWIFTv2Middleware] pod %s has secondary interface : %v", podInfo.Name(), req.SecondaryInterfacesExist) return types.Success, "" } @@ -69,6 +72,7 @@ func (m *SWIFTv2Middleware) GetIPConfig(ctx context.Context, podInfo cns.PodInfo if mtpnc.Status.PrimaryIP == "" || mtpnc.Status.MacAddress == "" || mtpnc.Status.NCID == "" || mtpnc.Status.GatewayIP == "" { return cns.PodIpInfo{}, errMTPNCNotReady } + logger.Debugf("[SWIFTv2Middleware] mtpnc for pod %s is : %+v", podInfo.Name(), mtpnc) // Parse MTPNC primaryIP to get the IP address and prefix length p, err := netip.ParsePrefix(mtpnc.Status.PrimaryIP) if err != nil { @@ -96,6 +100,7 @@ func (m *SWIFTv2Middleware) GetIPConfig(ctx context.Context, podInfo cns.PodInfo // SetRoutes sets the routes for podIPInfo used in SWIFT V2 scenario. func (m *SWIFTv2Middleware) SetRoutes(podIPInfo *cns.PodIpInfo) error { + logger.Debugf("[SWIFTv2Middleware] set routes for pod with nic type : %s", podIPInfo.NICType) podIPInfo.Routes = []cns.Route{} switch podIPInfo.NICType { case cns.DelegatedVMNIC: From 5660542910653f6464c979b1e9116a520fd5e10e Mon Sep 17 00:00:00 2001 From: Quang Nguyen Date: Fri, 27 Oct 2023 11:51:58 -0400 Subject: [PATCH 76/85] fix: log debugf to printf --- cns/middlewares/swiftV2.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cns/middlewares/swiftV2.go b/cns/middlewares/swiftV2.go index 20b3b47b6b..dac4591baa 100644 --- a/cns/middlewares/swiftV2.go +++ b/cns/middlewares/swiftV2.go @@ -39,7 +39,7 @@ type SWIFTv2Middleware struct { func (m *SWIFTv2Middleware) ValidateIPConfigsRequest(ctx context.Context, req *cns.IPConfigsRequest) (respCode types.ResponseCode, message string) { // Retrieve the pod from the cluster podInfo, err := cns.UnmarshalPodInfo(req.OrchestratorContext) - logger.Debugf("[SWIFTv2Middleware] validate ipconfigs request for pod %s", podInfo.Name()) + logger.Printf("[SWIFTv2Middleware] validate ipconfigs request for pod %s", podInfo.Name()) if err != nil { errBuf := fmt.Sprintf("unmarshalling pod info from ipconfigs request %v failed with error %v", req, err) return types.UnexpectedError, errBuf @@ -55,7 +55,7 @@ func (m *SWIFTv2Middleware) ValidateIPConfigsRequest(ctx context.Context, req *c if _, ok := pod.Labels[configuration.LabelPodSwiftV2]; ok { req.SecondaryInterfacesExist = true } - logger.Debugf("[SWIFTv2Middleware] pod %s has secondary interface : %v", podInfo.Name(), req.SecondaryInterfacesExist) + logger.Printf("[SWIFTv2Middleware] pod %s has secondary interface : %v", podInfo.Name(), req.SecondaryInterfacesExist) return types.Success, "" } @@ -72,7 +72,7 @@ func (m *SWIFTv2Middleware) GetIPConfig(ctx context.Context, podInfo cns.PodInfo if mtpnc.Status.PrimaryIP == "" || mtpnc.Status.MacAddress == "" || mtpnc.Status.NCID == "" || mtpnc.Status.GatewayIP == "" { return cns.PodIpInfo{}, errMTPNCNotReady } - logger.Debugf("[SWIFTv2Middleware] mtpnc for pod %s is : %+v", podInfo.Name(), mtpnc) + logger.Printf("[SWIFTv2Middleware] mtpnc for pod %s is : %+v", podInfo.Name(), mtpnc) // Parse MTPNC primaryIP to get the IP address and prefix length p, err := netip.ParsePrefix(mtpnc.Status.PrimaryIP) if err != nil { @@ -100,7 +100,7 @@ func (m *SWIFTv2Middleware) GetIPConfig(ctx context.Context, podInfo cns.PodInfo // SetRoutes sets the routes for podIPInfo used in SWIFT V2 scenario. func (m *SWIFTv2Middleware) SetRoutes(podIPInfo *cns.PodIpInfo) error { - logger.Debugf("[SWIFTv2Middleware] set routes for pod with nic type : %s", podIPInfo.NICType) + logger.Printf("[SWIFTv2Middleware] set routes for pod with nic type : %s", podIPInfo.NICType) podIPInfo.Routes = []cns.Route{} switch podIPInfo.NICType { case cns.DelegatedVMNIC: From dc005cd6b3ee36dd978a2ebaa380a8b6f34982b8 Mon Sep 17 00:00:00 2001 From: Quang Nguyen Date: Fri, 27 Oct 2023 12:22:22 -0400 Subject: [PATCH 77/85] fix: add testmain to avoid nil pointer error for loggers --- cns/middlewares/swiftV2.go | 2 +- cns/middlewares/swiftV2_test.go | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/cns/middlewares/swiftV2.go b/cns/middlewares/swiftV2.go index dac4591baa..341da015e3 100644 --- a/cns/middlewares/swiftV2.go +++ b/cns/middlewares/swiftV2.go @@ -39,11 +39,11 @@ type SWIFTv2Middleware struct { func (m *SWIFTv2Middleware) ValidateIPConfigsRequest(ctx context.Context, req *cns.IPConfigsRequest) (respCode types.ResponseCode, message string) { // Retrieve the pod from the cluster podInfo, err := cns.UnmarshalPodInfo(req.OrchestratorContext) - logger.Printf("[SWIFTv2Middleware] validate ipconfigs request for pod %s", podInfo.Name()) if err != nil { errBuf := fmt.Sprintf("unmarshalling pod info from ipconfigs request %v failed with error %v", req, err) return types.UnexpectedError, errBuf } + logger.Printf("[SWIFTv2Middleware] validate ipconfigs request for pod %s", podInfo.Name()) podNamespacedName := k8stypes.NamespacedName{Namespace: podInfo.Namespace(), Name: podInfo.Name()} pod := v1.Pod{} if err := m.Cli.Get(ctx, podNamespacedName, &pod); err != nil { diff --git a/cns/middlewares/swiftV2_test.go b/cns/middlewares/swiftV2_test.go index 60b41221c6..941cd0fc29 100644 --- a/cns/middlewares/swiftV2_test.go +++ b/cns/middlewares/swiftV2_test.go @@ -7,6 +7,7 @@ import ( "github.com/Azure/azure-container-networking/cns" "github.com/Azure/azure-container-networking/cns/configuration" + "github.com/Azure/azure-container-networking/cns/logger" mock "github.com/Azure/azure-container-networking/cns/middlewares/mock" "github.com/Azure/azure-container-networking/cns/types" "gotest.tools/v3/assert" @@ -35,6 +36,11 @@ func unsetEnvVar() { os.Unsetenv(configuration.EnvNodeCIDRs) } +func TestMain(m *testing.M) { + logger.InitLogger("testlogs", 0, 0, "./") + m.Run() +} + func TestValidateMultitenantIPConfigsRequestSuccess(t *testing.T) { middleware := SWIFTv2Middleware{Cli: mock.NewMockClient()} From 94a8e2db1bd7b1553c39decb512dbb3b0fc8712f Mon Sep 17 00:00:00 2001 From: Quang Nguyen Date: Fri, 27 Oct 2023 15:26:20 -0400 Subject: [PATCH 78/85] Update azure-cns.yaml Signed-off-by: Quang Nguyen --- cns/azure-cns.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cns/azure-cns.yaml b/cns/azure-cns.yaml index 7975a0a9f6..12c3cd249e 100644 --- a/cns/azure-cns.yaml +++ b/cns/azure-cns.yaml @@ -1,4 +1,4 @@ - +apiVersion: v1 kind: RoleBinding metadata: name: nodeNetConfigEditorRoleBinding @@ -161,4 +161,4 @@ data: "ManageEndpointState": false, "ProgramSNATIPTables" : false } -# Toggle ManageEndpointState and ProgramSNATIPTables to true for delegated IPAM use case. \ No newline at end of file +# Toggle ManageEndpointState and ProgramSNATIPTables to true for delegated IPAM use case. From 2d305e16d852a69c6ff58416670d7ddb8ae69337 Mon Sep 17 00:00:00 2001 From: Quang Nguyen Date: Fri, 27 Oct 2023 17:09:57 -0400 Subject: [PATCH 79/85] fix: move parseCIDRs to a common package, use net/netip instead of net --- cns/middlewares/mock/mockClient.go | 14 ++++---- cns/middlewares/mock/mockSWIFTv2.go | 42 +++++++--------------- cns/middlewares/swiftV2.go | 24 +++---------- cns/middlewares/swiftV2_test.go | 14 ++++---- cns/middlewares/utils/utils.go | 26 ++++++++++++++ cns/middlewares/utils/utils_test.go | 56 +++++++++++++++++++++++++++++ cns/restserver/internalapi_test.go | 10 +++--- cns/restserver/ipam.go | 1 + cns/restserver/ipam_test.go | 10 +++--- 9 files changed, 123 insertions(+), 74 deletions(-) create mode 100644 cns/middlewares/utils/utils.go create mode 100644 cns/middlewares/utils/utils_test.go diff --git a/cns/middlewares/mock/mockClient.go b/cns/middlewares/mock/mockClient.go index b888c735fe..0baf31d1ce 100644 --- a/cns/middlewares/mock/mockClient.go +++ b/cns/middlewares/mock/mockClient.go @@ -1,4 +1,4 @@ -package middlewares +package mock import ( "context" @@ -15,15 +15,15 @@ var ( errMTPNCNotFound = errors.New("mtpnc not found") ) -// MockClient implements the client.Client interface for testing. We only care about Get, the rest is nil ops. -type MockClient struct { +// Client implements the client.Client interface for testing. We only care about Get, the rest is nil ops. +type Client struct { client.Client mtPodCache map[string]*v1.Pod mtpncCache map[string]*v1alpha1.MultitenantPodNetworkConfig } -// NewMockClient returns a new MockClient. -func NewMockClient() *MockClient { +// NewClient returns a new MockClient. +func NewClient() *Client { testPod1 := v1.Pod{} testPod1.Labels = make(map[string]string) testPod1.Labels[configuration.LabelPodSwiftV2] = "true" @@ -36,7 +36,7 @@ func NewMockClient() *MockClient { testMTPNC3 := v1alpha1.MultitenantPodNetworkConfig{} - return &MockClient{ + return &Client{ mtPodCache: map[string]*v1.Pod{"testpod1namespace/testpod1": &testPod1}, mtpncCache: map[string]*v1alpha1.MultitenantPodNetworkConfig{ "testpod1namespace/testpod1": &testMTPNC1, @@ -46,7 +46,7 @@ func NewMockClient() *MockClient { } // Get implements client.Client.Get. -func (c *MockClient) Get(_ context.Context, key client.ObjectKey, obj client.Object, _ ...client.GetOption) error { +func (c *Client) Get(_ context.Context, key client.ObjectKey, obj client.Object, _ ...client.GetOption) error { switch o := obj.(type) { case *v1.Pod: if pod, ok := c.mtPodCache[key.String()]; ok { diff --git a/cns/middlewares/mock/mockSWIFTv2.go b/cns/middlewares/mock/mockSWIFTv2.go index 9602521773..e8e946b12a 100644 --- a/cns/middlewares/mock/mockSWIFTv2.go +++ b/cns/middlewares/mock/mockSWIFTv2.go @@ -1,4 +1,4 @@ -package middlewares +package mock import ( "context" @@ -6,10 +6,10 @@ import ( "fmt" "net" "os" - "strings" "github.com/Azure/azure-container-networking/cns" "github.com/Azure/azure-container-networking/cns/configuration" + "github.com/Azure/azure-container-networking/cns/middlewares/utils" "github.com/Azure/azure-container-networking/cns/types" "github.com/Azure/azure-container-networking/crd/multitenancy/api/v1alpha1" v1 "k8s.io/api/core/v1" @@ -28,38 +28,38 @@ const ( overlayGatewayV6 = "fe80::1234:5678:9abc" ) -type MockSWIFTv2Middleware struct { +type SWIFTv2Middleware struct { mtPodState map[string]*v1.Pod mtpncState map[string]*v1alpha1.MultitenantPodNetworkConfig } -func NewMockSWIFTv2Middleware() *MockSWIFTv2Middleware { +func NewMockSWIFTv2Middleware() *SWIFTv2Middleware { testPod1 := v1.Pod{} testPod1.Labels = make(map[string]string) testPod1.Labels[configuration.LabelPodSwiftV2] = "true" testMTPNC1 := v1alpha1.MultitenantPodNetworkConfig{} - return &MockSWIFTv2Middleware{ + return &SWIFTv2Middleware{ mtPodState: map[string]*v1.Pod{"testpod1namespace/testpod1": &testPod1}, mtpncState: map[string]*v1alpha1.MultitenantPodNetworkConfig{"testpod1namespace/testpod1": &testMTPNC1}, } } -func (m *MockSWIFTv2Middleware) SetMTPNCReady() { +func (m *SWIFTv2Middleware) SetMTPNCReady() { m.mtpncState["testpod1namespace/testpod1"].Status.PrimaryIP = "192.168.0.1" m.mtpncState["testpod1namespace/testpod1"].Status.MacAddress = "00:00:00:00:00:00" m.mtpncState["testpod1namespace/testpod1"].Status.GatewayIP = "10.0.0.1" m.mtpncState["testpod1namespace/testpod1"].Status.NCID = "testncid" } -func (m *MockSWIFTv2Middleware) SetEnvVar() { +func (m *SWIFTv2Middleware) SetEnvVar() { os.Setenv(configuration.EnvPodCIDRs, "10.0.1.10/24") os.Setenv(configuration.EnvServiceCIDRs, "10.0.2.10/24") os.Setenv(configuration.EnvNodeCIDRs, "10.0.3.10/24") } -func (m *MockSWIFTv2Middleware) UnsetEnvVar() error { +func (m *SWIFTv2Middleware) UnsetEnvVar() error { if err := os.Unsetenv(configuration.EnvPodCIDRs); err != nil { return fmt.Errorf("failed to unset env var %s : %w", configuration.EnvPodCIDRs, err) } @@ -74,7 +74,7 @@ func (m *MockSWIFTv2Middleware) UnsetEnvVar() error { // validateMultitenantIPConfigsRequest validates if pod is multitenant // nolint -func (m *MockSWIFTv2Middleware) ValidateIPConfigsRequest(_ context.Context, req *cns.IPConfigsRequest) (respCode types.ResponseCode, message string) { +func (m *SWIFTv2Middleware) ValidateIPConfigsRequest(_ context.Context, req *cns.IPConfigsRequest) (respCode types.ResponseCode, message string) { // Retrieve the pod from the cluster podInfo, err := cns.UnmarshalPodInfo(req.OrchestratorContext) if err != nil { @@ -96,7 +96,7 @@ func (m *MockSWIFTv2Middleware) ValidateIPConfigsRequest(_ context.Context, req // GetSWIFTv2IPConfig(podInfo PodInfo) (*PodIpInfo, error) // GetMultitenantIPConfig returns the IP config for a multitenant pod from the MTPNC CRD -func (m *MockSWIFTv2Middleware) GetIPConfig(_ context.Context, podInfo cns.PodInfo) (cns.PodIpInfo, error) { +func (m *SWIFTv2Middleware) GetIPConfig(_ context.Context, podInfo cns.PodInfo) (cns.PodIpInfo, error) { // Check if the MTPNC CRD exists for the pod, if not, return error mtpncNamespacedName := k8types.NamespacedName{Namespace: podInfo.Namespace(), Name: podInfo.Name()} mtpnc, ok := m.mtpncState[mtpncNamespacedName.String()] @@ -119,7 +119,7 @@ func (m *MockSWIFTv2Middleware) GetIPConfig(_ context.Context, podInfo cns.PodIn return podIPInfo, nil } -func (m *MockSWIFTv2Middleware) SetRoutes(podIPInfo *cns.PodIpInfo) error { +func (m *SWIFTv2Middleware) SetRoutes(podIPInfo *cns.PodIpInfo) error { podIPInfo.Routes = []cns.Route{} switch podIPInfo.NICType { case cns.DelegatedVMNIC: @@ -133,7 +133,7 @@ func (m *MockSWIFTv2Middleware) SetRoutes(podIPInfo *cns.PodIpInfo) error { if err != nil { return fmt.Errorf("failed to get podCIDRs from env : %w", err) } - podCIDRsV4, podCIDRv6, err := parseCIDRs(podCIDRs) + podCIDRsV4, podCIDRv6, err := utils.ParseCIDRs(podCIDRs) if err != nil { return fmt.Errorf("failed to parse podCIDRs : %w", err) } @@ -142,7 +142,7 @@ func (m *MockSWIFTv2Middleware) SetRoutes(podIPInfo *cns.PodIpInfo) error { if err != nil { return fmt.Errorf("failed to get serviceCIDRs from env : %w", err) } - serviceCIDRsV4, serviceCIDRsV6, err := parseCIDRs(serviceCIDRs) + serviceCIDRsV4, serviceCIDRsV6, err := utils.ParseCIDRs(serviceCIDRs) if err != nil { return fmt.Errorf("failed to parse serviceCIDRs : %w", err) } @@ -188,19 +188,3 @@ func (m *MockSWIFTv2Middleware) SetRoutes(podIPInfo *cns.PodIpInfo) error { } return nil } - -// parseCIDRs parses the semicolons separated CIDRs string and returns the IPv4 and IPv6 CIDRs. -func parseCIDRs(cidrs string) (v4IPs, v6IPs []string, err error) { - for _, cidr := range strings.Split(cidrs, ",") { - ip, _, err := net.ParseCIDR(cidr) - if err != nil { - return nil, nil, fmt.Errorf("failed to parse cidr %s : %w", cidr, err) - } - if ip.To4() != nil { - v4IPs = append(v4IPs, cidr) - } else { - v6IPs = append(v6IPs, cidr) - } - } - return v4IPs, v6IPs, nil -} diff --git a/cns/middlewares/swiftV2.go b/cns/middlewares/swiftV2.go index 341da015e3..8c15a20395 100644 --- a/cns/middlewares/swiftV2.go +++ b/cns/middlewares/swiftV2.go @@ -6,11 +6,11 @@ import ( "fmt" "net" "net/netip" - "strings" "github.com/Azure/azure-container-networking/cns" "github.com/Azure/azure-container-networking/cns/configuration" "github.com/Azure/azure-container-networking/cns/logger" + "github.com/Azure/azure-container-networking/cns/middlewares/utils" "github.com/Azure/azure-container-networking/cns/types" "github.com/Azure/azure-container-networking/crd/multitenancy/api/v1alpha1" v1 "k8s.io/api/core/v1" @@ -115,7 +115,7 @@ func (m *SWIFTv2Middleware) SetRoutes(podIPInfo *cns.PodIpInfo) error { if err != nil { return fmt.Errorf("failed to get nodeCIDR from env : %w", err) } - nodeCIDRsv4, nodeCIDRsv6, err := parseCIDRs(nodeCIDRs) + nodeCIDRsv4, nodeCIDRsv6, err := utils.ParseCIDRs(nodeCIDRs) if err != nil { return fmt.Errorf("failed to parse nodeCIDRs : %w", err) } @@ -125,7 +125,7 @@ func (m *SWIFTv2Middleware) SetRoutes(podIPInfo *cns.PodIpInfo) error { if err != nil { return fmt.Errorf("failed to get podCIDRs from env : %w", err) } - podCIDRsV4, podCIDRv6, err := parseCIDRs(podCIDRs) + podCIDRsV4, podCIDRv6, err := utils.ParseCIDRs(podCIDRs) if err != nil { return fmt.Errorf("failed to parse podCIDRs : %w", err) } @@ -135,7 +135,7 @@ func (m *SWIFTv2Middleware) SetRoutes(podIPInfo *cns.PodIpInfo) error { if err != nil { return fmt.Errorf("failed to get serviceCIDRs from env : %w", err) } - serviceCIDRsV4, serviceCIDRsV6, err := parseCIDRs(serviceCIDRs) + serviceCIDRsV4, serviceCIDRsV6, err := utils.ParseCIDRs(serviceCIDRs) if err != nil { return fmt.Errorf("failed to parse serviceCIDRs : %w", err) } @@ -197,19 +197,3 @@ func (m *SWIFTv2Middleware) SetRoutes(podIPInfo *cns.PodIpInfo) error { } return nil } - -// parseCIDRs parses the semicolons separated CIDRs string and returns the IPv4 and IPv6 CIDRs. -func parseCIDRs(cidrs string) (v4IPs, v6IPs []string, err error) { - for _, cidr := range strings.Split(cidrs, ",") { - ip, _, err := net.ParseCIDR(cidr) - if err != nil { - return nil, nil, fmt.Errorf("failed to parse cidr %s : %w", cidr, err) - } - if ip.To4() != nil { - v4IPs = append(v4IPs, cidr) - } else { - v6IPs = append(v6IPs, cidr) - } - } - return v4IPs, v6IPs, nil -} diff --git a/cns/middlewares/swiftV2_test.go b/cns/middlewares/swiftV2_test.go index 941cd0fc29..404bdecd74 100644 --- a/cns/middlewares/swiftV2_test.go +++ b/cns/middlewares/swiftV2_test.go @@ -8,7 +8,7 @@ import ( "github.com/Azure/azure-container-networking/cns" "github.com/Azure/azure-container-networking/cns/configuration" "github.com/Azure/azure-container-networking/cns/logger" - mock "github.com/Azure/azure-container-networking/cns/middlewares/mock" + "github.com/Azure/azure-container-networking/cns/middlewares/mock" "github.com/Azure/azure-container-networking/cns/types" "gotest.tools/v3/assert" ) @@ -42,7 +42,7 @@ func TestMain(m *testing.M) { } func TestValidateMultitenantIPConfigsRequestSuccess(t *testing.T) { - middleware := SWIFTv2Middleware{Cli: mock.NewMockClient()} + middleware := SWIFTv2Middleware{Cli: mock.NewClient()} happyReq := &cns.IPConfigsRequest{ PodInterfaceID: testPod1Info.InterfaceID(), @@ -59,7 +59,7 @@ func TestValidateMultitenantIPConfigsRequestSuccess(t *testing.T) { } func TestValidateMultitenantIPConfigsRequestFailure(t *testing.T) { - middleware := SWIFTv2Middleware{Cli: mock.NewMockClient()} + middleware := SWIFTv2Middleware{Cli: mock.NewClient()} // Fail to unmarshal pod info test failReq := &cns.IPConfigsRequest{ @@ -85,7 +85,7 @@ func TestGetSWIFTv2IPConfigSuccess(t *testing.T) { setEnvVar() defer unsetEnvVar() - middleware := SWIFTv2Middleware{Cli: mock.NewMockClient()} + middleware := SWIFTv2Middleware{Cli: mock.NewClient()} ipInfo, err := middleware.GetIPConfig(context.TODO(), testPod1Info) assert.Equal(t, err, nil) @@ -94,7 +94,7 @@ func TestGetSWIFTv2IPConfigSuccess(t *testing.T) { } func TestGetSWIFTv2IPConfigFailure(t *testing.T) { - middleware := SWIFTv2Middleware{Cli: mock.NewMockClient()} + middleware := SWIFTv2Middleware{Cli: mock.NewClient()} // Pod's MTPNC doesn't exist in cache test _, err := middleware.GetIPConfig(context.TODO(), testPod2Info) @@ -106,7 +106,7 @@ func TestGetSWIFTv2IPConfigFailure(t *testing.T) { } func TestSetRoutesSuccess(t *testing.T) { - middleware := SWIFTv2Middleware{Cli: mock.NewMockClient()} + middleware := SWIFTv2Middleware{Cli: mock.NewClient()} setEnvVar() defer unsetEnvVar() podIPInfo := []cns.PodIpInfo{ @@ -208,7 +208,7 @@ func TestSetRoutesSuccess(t *testing.T) { func TestSetRoutesFailure(t *testing.T) { // Failure due to env var not set - middleware := SWIFTv2Middleware{Cli: mock.NewMockClient()} + middleware := SWIFTv2Middleware{Cli: mock.NewClient()} podIPInfo := []cns.PodIpInfo{ { PodIPConfig: cns.IPSubnet{ diff --git a/cns/middlewares/utils/utils.go b/cns/middlewares/utils/utils.go new file mode 100644 index 0000000000..19766a2c39 --- /dev/null +++ b/cns/middlewares/utils/utils.go @@ -0,0 +1,26 @@ +package utils + +import ( + "fmt" + "net/netip" + "strings" +) + +// ParseCIDRs parses the semicolons separated CIDRs string and returns the IPv4 and IPv6 CIDRs. +func ParseCIDRs(cidrs string) (v4IPs, v6IPs []string, err error) { + v4IPs = []string{} + v6IPs = []string{} + for _, cidr := range strings.Split(cidrs, ",") { + p, err := netip.ParsePrefix(cidr) + if err != nil { + return nil, nil, fmt.Errorf("failed to parse CIDR %s : %w", cidr, err) + } + ip := p.Addr() + if ip.Is4() { + v4IPs = append(v4IPs, cidr) + } else { + v6IPs = append(v6IPs, cidr) + } + } + return v4IPs, v6IPs, nil +} diff --git a/cns/middlewares/utils/utils_test.go b/cns/middlewares/utils/utils_test.go new file mode 100644 index 0000000000..aaa85c024f --- /dev/null +++ b/cns/middlewares/utils/utils_test.go @@ -0,0 +1,56 @@ +package utils + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestParseCIDRs(t *testing.T) { + // Test valid IPv4 CIDR + cidr := "192.168.0.0/16" + expectedV4 := []string{"192.168.0.0/16"} + expectedV6 := []string{} + resultV4, resultV6, err := ParseCIDRs(cidr) + if err != nil { + t.Errorf("Unexpected error: %v", err) + } + assert.Equal(t, expectedV4, resultV4) + assert.Equal(t, expectedV6, resultV6) + + // Test valid IPv6 CIDR + cidr = "2001:db8::/32" + expectedV4 = []string{} + expectedV6 = []string{"2001:db8::/32"} + resultV4, resultV6, err = ParseCIDRs(cidr) + if err != nil { + t.Errorf("Unexpected error: %v", err) + } + assert.Equal(t, expectedV4, resultV4) + assert.Equal(t, expectedV6, resultV6) + + // Test multiple valid CIDRs + cidrs := "192.168.0.0/16,10.0.0.0/8,2001:db8::/32" + expectedV4 = []string{"192.168.0.0/16", "10.0.0.0/8"} + expectedV6 = []string{"2001:db8::/32"} + resultV4, resultV6, err = ParseCIDRs(cidrs) + if err != nil { + t.Errorf("Unexpected error: %v", err) + } + assert.Equal(t, expectedV4, resultV4) + assert.Equal(t, expectedV6, resultV6) + + // Test invalid CIDR + cidr = "192.168.0.0/33" + _, _, err = ParseCIDRs(cidr) + if err == nil { + t.Errorf("Expected error but got nil") + } + + // Test invalid CIDRs + cidrs = "192.168.0.0/33,10.0.0.0/8,2001:db8::/33" + _, _, err = ParseCIDRs(cidrs) + if err == nil { + t.Errorf("Expected error but got nil") + } +} diff --git a/cns/restserver/internalapi_test.go b/cns/restserver/internalapi_test.go index 34ec365d9a..0d26e0f2f0 100644 --- a/cns/restserver/internalapi_test.go +++ b/cns/restserver/internalapi_test.go @@ -29,12 +29,10 @@ import ( ) const ( - primaryIp = "10.0.0.5" + primaryIP = "10.0.0.5" SWIFTv2IP = "192.168.0.1" SWIFTv2MAC = "00:00:00:00:00:00" - SWIFTv2GatewayIP = "10.0.0.1" - SWIFTv2NCID = "testncid" - gatewayIp = "10.0.0.1" + gatewayIP = "10.0.0.1" subnetPrfixLength = 24 dockerContainerType = cns.Docker releasePercent = 50 @@ -920,9 +918,9 @@ func validateNetworkRequest(t *testing.T, req cns.CreateNetworkContainerRequest) func generateNetworkContainerRequest(secondaryIps map[string]cns.SecondaryIPConfig, ncID, ncVersion string) *cns.CreateNetworkContainerRequest { var ipConfig cns.IPConfiguration ipConfig.DNSServers = dnsservers - ipConfig.GatewayIPAddress = gatewayIp + ipConfig.GatewayIPAddress = gatewayIP var ipSubnet cns.IPSubnet - ipSubnet.IPAddress = primaryIp + ipSubnet.IPAddress = primaryIP ipSubnet.PrefixLength = subnetPrfixLength ipConfig.IPSubnet = ipSubnet diff --git a/cns/restserver/ipam.go b/cns/restserver/ipam.go index 5bb9dffda1..09c24578bf 100644 --- a/cns/restserver/ipam.go +++ b/cns/restserver/ipam.go @@ -24,6 +24,7 @@ var ( ErrNoNCs = errors.New("No NCs found in the CNS internal state") ) +// requestIPConfigHandlerHelper validates the request, assign IPs and return the IPConfigs func (service *HTTPRestService) requestIPConfigHandlerHelper(ctx context.Context, ipconfigsRequest cns.IPConfigsRequest) (*cns.IPConfigsResponse, error) { // For SWIFT v2 scenario, the validator function will also modify the ipconfigsRequest. podInfo, returnCode, returnMessage := service.validateIPConfigsRequest(ctx, ipconfigsRequest) diff --git a/cns/restserver/ipam_test.go b/cns/restserver/ipam_test.go index aac099e5a5..3518199a24 100644 --- a/cns/restserver/ipam_test.go +++ b/cns/restserver/ipam_test.go @@ -14,7 +14,7 @@ import ( "github.com/Azure/azure-container-networking/cns" "github.com/Azure/azure-container-networking/cns/common" "github.com/Azure/azure-container-networking/cns/fakes" - middlewares "github.com/Azure/azure-container-networking/cns/middlewares/mock" + "github.com/Azure/azure-container-networking/cns/middlewares/mock" "github.com/Azure/azure-container-networking/cns/types" "github.com/Azure/azure-container-networking/crd/nodenetworkconfig/api/v1alpha" "github.com/Azure/azure-container-networking/store" @@ -99,10 +99,10 @@ func requestIPAddressAndGetState(t *testing.T, req cns.IPConfigsRequest) ([]cns. } for i := range podIPInfo { - assert.Equal(t, primaryIp, podIPInfo[i].NetworkContainerPrimaryIPConfig.IPSubnet.IPAddress) + assert.Equal(t, primaryIP, podIPInfo[i].NetworkContainerPrimaryIPConfig.IPSubnet.IPAddress) assert.Equal(t, subnetPrfixLength, int(podIPInfo[i].NetworkContainerPrimaryIPConfig.IPSubnet.PrefixLength)) assert.Equal(t, dnsservers, podIPInfo[i].NetworkContainerPrimaryIPConfig.DNSServers) - assert.Equal(t, gatewayIp, podIPInfo[i].NetworkContainerPrimaryIPConfig.GatewayIPAddress) + assert.Equal(t, gatewayIP, podIPInfo[i].NetworkContainerPrimaryIPConfig.GatewayIPAddress) assert.Equal(t, subnetPrfixLength, int(podIPInfo[i].PodIPConfig.PrefixLength)) assert.Equal(t, fakes.HostPrimaryIP, podIPInfo[i].HostPrimaryIPInfo.PrimaryIP) assert.Equal(t, fakes.HostSubnet, podIPInfo[i].HostPrimaryIPInfo.Subnet) @@ -1535,7 +1535,7 @@ func TestIPAMFailToRequestPartialIPsInPool(t *testing.T) { func TestIPAMGetSWIFTv2IPSuccess(t *testing.T) { svc := getTestService() - middleware := middlewares.NewMockSWIFTv2Middleware() + middleware := mock.NewMockSWIFTv2Middleware() svc.AttachSWIFTv2Middleware(middleware) middleware.SetEnvVar() @@ -1597,7 +1597,7 @@ func TestIPAMGetSWIFTv2IPSuccess(t *testing.T) { func TestIPAMGetSWIFTv2IPFailure(t *testing.T) { svc := getTestService() - middleware := middlewares.NewMockSWIFTv2Middleware() + middleware := mock.NewMockSWIFTv2Middleware() svc.AttachSWIFTv2Middleware(middleware) ncStates := []ncState{ { From 54aa8be39abb23d80ce9424de18ab1e36aca0e84 Mon Sep 17 00:00:00 2001 From: Quang Nguyen Date: Sat, 28 Oct 2023 13:57:31 -0400 Subject: [PATCH 80/85] fix: exhaustive all switch case for nic type --- cns/middlewares/swiftV2.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cns/middlewares/swiftV2.go b/cns/middlewares/swiftV2.go index 8c15a20395..e901fa57b3 100644 --- a/cns/middlewares/swiftV2.go +++ b/cns/middlewares/swiftV2.go @@ -192,6 +192,7 @@ func (m *SWIFTv2Middleware) SetRoutes(podIPInfo *cns.PodIpInfo) error { } } podIPInfo.SkipDefaultRoutes = true + case cns.BackendNIC: default: return errInvalidSWIFTv2NICType } From 86b9e7dc028fd931adcc84863ea7b6577b53ad09 Mon Sep 17 00:00:00 2001 From: Quang Nguyen Date: Sat, 28 Oct 2023 13:58:47 -0400 Subject: [PATCH 81/85] fix: exhaustive all switch case for nic type --- cns/middlewares/mock/mockSWIFTv2.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cns/middlewares/mock/mockSWIFTv2.go b/cns/middlewares/mock/mockSWIFTv2.go index e8e946b12a..f260049597 100644 --- a/cns/middlewares/mock/mockSWIFTv2.go +++ b/cns/middlewares/mock/mockSWIFTv2.go @@ -183,6 +183,7 @@ func (m *SWIFTv2Middleware) SetRoutes(podIPInfo *cns.PodIpInfo) error { podIPInfo.Routes = append(podIPInfo.Routes, serviceCIDRv6Route) } } + case cns.BackendNIC: default: return errInvalidSWIFTv2NICType } From 8fed7b92e3be90d6ad736c0ebee0232e825285c5 Mon Sep 17 00:00:00 2001 From: Quang Nguyen Date: Mon, 30 Oct 2023 13:22:04 -0400 Subject: [PATCH 82/85] refractor: change fmt.Errorf to errors.Wrapf --- cns/middlewares/mock/mockClient.go | 10 ++++---- cns/middlewares/mock/mockSWIFTv2.go | 40 ++++++++++++++++++++++++----- cns/middlewares/swiftV2.go | 29 ++++++++++----------- cns/middlewares/swiftV2_test.go | 2 +- 4 files changed, 54 insertions(+), 27 deletions(-) diff --git a/cns/middlewares/mock/mockClient.go b/cns/middlewares/mock/mockClient.go index 0baf31d1ce..bf6a14deeb 100644 --- a/cns/middlewares/mock/mockClient.go +++ b/cns/middlewares/mock/mockClient.go @@ -2,17 +2,17 @@ package mock import ( "context" - "errors" "github.com/Azure/azure-container-networking/cns/configuration" "github.com/Azure/azure-container-networking/crd/multitenancy/api/v1alpha1" + "github.com/pkg/errors" v1 "k8s.io/api/core/v1" "sigs.k8s.io/controller-runtime/pkg/client" ) var ( - errPodNotFound = errors.New("pod not found") - errMTPNCNotFound = errors.New("mtpnc not found") + ErrPodNotFound = errors.New("pod not found") + ErrMTPNCNotFound = errors.New("mtpnc not found") ) // Client implements the client.Client interface for testing. We only care about Get, the rest is nil ops. @@ -52,13 +52,13 @@ func (c *Client) Get(_ context.Context, key client.ObjectKey, obj client.Object, if pod, ok := c.mtPodCache[key.String()]; ok { *o = *pod } else { - return errPodNotFound + return ErrPodNotFound } case *v1alpha1.MultitenantPodNetworkConfig: if mtpnc, ok := c.mtpncCache[key.String()]; ok { *o = *mtpnc } else { - return errMTPNCNotFound + return ErrMTPNCNotFound } } return nil diff --git a/cns/middlewares/mock/mockSWIFTv2.go b/cns/middlewares/mock/mockSWIFTv2.go index f260049597..9d35fdf2e6 100644 --- a/cns/middlewares/mock/mockSWIFTv2.go +++ b/cns/middlewares/mock/mockSWIFTv2.go @@ -2,7 +2,6 @@ package mock import ( "context" - "errors" "fmt" "net" "os" @@ -12,6 +11,7 @@ import ( "github.com/Azure/azure-container-networking/cns/middlewares/utils" "github.com/Azure/azure-container-networking/cns/types" "github.com/Azure/azure-container-networking/crd/multitenancy/api/v1alpha1" + "github.com/pkg/errors" v1 "k8s.io/api/core/v1" k8types "k8s.io/apimachinery/pkg/types" ) @@ -129,22 +129,34 @@ func (m *SWIFTv2Middleware) SetRoutes(podIPInfo *cns.PodIpInfo) error { } podIPInfo.Routes = []cns.Route{route} case cns.InfraNIC: + // Get and parse nodeCIDRs from env + nodeCIDRs, err := configuration.NodeCIDRs() + if err != nil { + return errors.Wrapf(err, "failed to get nodeCIDR from env") + } + nodeCIDRsv4, nodeCIDRsv6, err := utils.ParseCIDRs(nodeCIDRs) + if err != nil { + return errors.Wrapf(err, "failed to parse nodeCIDRs") + } + + // Get and parse podCIDRs from env podCIDRs, err := configuration.PodCIDRs() if err != nil { - return fmt.Errorf("failed to get podCIDRs from env : %w", err) + return errors.Wrapf(err, "failed to get podCIDRs from env") } podCIDRsV4, podCIDRv6, err := utils.ParseCIDRs(podCIDRs) if err != nil { - return fmt.Errorf("failed to parse podCIDRs : %w", err) + return errors.Wrapf(err, "failed to parse podCIDRs") } + // Get and parse serviceCIDRs from env serviceCIDRs, err := configuration.ServiceCIDRs() if err != nil { - return fmt.Errorf("failed to get serviceCIDRs from env : %w", err) + return errors.Wrapf(err, "failed to get serviceCIDRs from env") } serviceCIDRsV4, serviceCIDRsV6, err := utils.ParseCIDRs(serviceCIDRs) if err != nil { - return fmt.Errorf("failed to parse serviceCIDRs : %w", err) + return errors.Wrapf(err, "failed to parse serviceCIDRs") } // Check if the podIPInfo is IPv4 or IPv6 if net.ParseIP(podIPInfo.PodIPConfig.IPAddress).To4() != nil { @@ -164,7 +176,14 @@ func (m *SWIFTv2Middleware) SetRoutes(podIPInfo *cns.PodIpInfo) error { } podIPInfo.Routes = append(podIPInfo.Routes, serviceCIDRv4Route) } - + // route for IPv4 nodeCIDR traffic + for _, nodeCIDRv4 := range nodeCIDRsv4 { + nodeCIDRv4Route := cns.Route{ + IPAddress: nodeCIDRv4, + GatewayIPAddress: overlayGatewayv4, + } + podIPInfo.Routes = append(podIPInfo.Routes, nodeCIDRv4Route) + } } else { // routes for IPv6 podCIDR traffic for _, podCIDRv6 := range podCIDRv6 { @@ -182,7 +201,16 @@ func (m *SWIFTv2Middleware) SetRoutes(podIPInfo *cns.PodIpInfo) error { } podIPInfo.Routes = append(podIPInfo.Routes, serviceCIDRv6Route) } + // route for IPv6 nodeCIDR traffic + for _, nodeCIDRv6 := range nodeCIDRsv6 { + nodeCIDRv6Route := cns.Route{ + IPAddress: nodeCIDRv6, + GatewayIPAddress: overlayGatewayV6, + } + podIPInfo.Routes = append(podIPInfo.Routes, nodeCIDRv6Route) + } } + podIPInfo.SkipDefaultRoutes = true case cns.BackendNIC: default: return errInvalidSWIFTv2NICType diff --git a/cns/middlewares/swiftV2.go b/cns/middlewares/swiftV2.go index e901fa57b3..3eb1ef79ac 100644 --- a/cns/middlewares/swiftV2.go +++ b/cns/middlewares/swiftV2.go @@ -2,8 +2,6 @@ package middlewares import ( "context" - "errors" - "fmt" "net" "net/netip" @@ -13,6 +11,7 @@ import ( "github.com/Azure/azure-container-networking/cns/middlewares/utils" "github.com/Azure/azure-container-networking/cns/types" "github.com/Azure/azure-container-networking/crd/multitenancy/api/v1alpha1" + "github.com/pkg/errors" v1 "k8s.io/api/core/v1" k8stypes "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" @@ -40,15 +39,15 @@ func (m *SWIFTv2Middleware) ValidateIPConfigsRequest(ctx context.Context, req *c // Retrieve the pod from the cluster podInfo, err := cns.UnmarshalPodInfo(req.OrchestratorContext) if err != nil { - errBuf := fmt.Sprintf("unmarshalling pod info from ipconfigs request %v failed with error %v", req, err) - return types.UnexpectedError, errBuf + errBuf := errors.Wrapf(err, "failed to unmarshalling pod info from ipconfigs request %v", req) + return types.UnexpectedError, errBuf.Error() } logger.Printf("[SWIFTv2Middleware] validate ipconfigs request for pod %s", podInfo.Name()) podNamespacedName := k8stypes.NamespacedName{Namespace: podInfo.Namespace(), Name: podInfo.Name()} pod := v1.Pod{} if err := m.Cli.Get(ctx, podNamespacedName, &pod); err != nil { - errBuf := fmt.Sprintf("failed to get pod %v with error %v", podNamespacedName, err) - return types.UnexpectedError, errBuf + errBuf := errors.Wrapf(err, "failed to get pod %v", podNamespacedName) + return types.UnexpectedError, errBuf.Error() } // check the pod labels for Swift V2, set the request's SecondaryInterfaceSet flag to true. @@ -65,7 +64,7 @@ func (m *SWIFTv2Middleware) GetIPConfig(ctx context.Context, podInfo cns.PodInfo mtpnc := v1alpha1.MultitenantPodNetworkConfig{} mtpncNamespacedName := k8stypes.NamespacedName{Namespace: podInfo.Namespace(), Name: podInfo.Name()} if err := m.Cli.Get(ctx, mtpncNamespacedName, &mtpnc); err != nil { - return cns.PodIpInfo{}, fmt.Errorf("failed to get pod's mtpnc from cache : %w", err) + return cns.PodIpInfo{}, errors.Wrapf(err, "failed to get pod's mtpnc from cache") } // Check if the MTPNC CRD is ready. If one of the fields is empty, return error @@ -76,13 +75,13 @@ func (m *SWIFTv2Middleware) GetIPConfig(ctx context.Context, podInfo cns.PodInfo // Parse MTPNC primaryIP to get the IP address and prefix length p, err := netip.ParsePrefix(mtpnc.Status.PrimaryIP) if err != nil { - return cns.PodIpInfo{}, fmt.Errorf("failed to parse MTPNC primaryIP %s : %w", mtpnc.Status.PrimaryIP, err) + return cns.PodIpInfo{}, errors.Wrapf(err, "failed to parse mtpnc primaryIP %s", mtpnc.Status.PrimaryIP) } // Get the IP address and prefix length ip := p.Addr() prefixSize := p.Bits() if prefixSize != prefixLength { - return cns.PodIpInfo{}, fmt.Errorf("%w, MTPNC primaryIP prefix length is %d", errInvalidMTPNCPrefixLength, prefixSize) + return cns.PodIpInfo{}, errors.Wrapf(errInvalidMTPNCPrefixLength, "mtpnc primaryIP prefix length is %d", prefixSize) } podIPInfo := cns.PodIpInfo{ PodIPConfig: cns.IPSubnet{ @@ -113,31 +112,31 @@ func (m *SWIFTv2Middleware) SetRoutes(podIPInfo *cns.PodIpInfo) error { // Get and parse nodeCIDRs from env nodeCIDRs, err := configuration.NodeCIDRs() if err != nil { - return fmt.Errorf("failed to get nodeCIDR from env : %w", err) + return errors.Wrapf(err, "failed to get nodeCIDR from env") } nodeCIDRsv4, nodeCIDRsv6, err := utils.ParseCIDRs(nodeCIDRs) if err != nil { - return fmt.Errorf("failed to parse nodeCIDRs : %w", err) + return errors.Wrapf(err, "failed to parse nodeCIDRs") } // Get and parse podCIDRs from env podCIDRs, err := configuration.PodCIDRs() if err != nil { - return fmt.Errorf("failed to get podCIDRs from env : %w", err) + return errors.Wrapf(err, "failed to get podCIDRs from env") } podCIDRsV4, podCIDRv6, err := utils.ParseCIDRs(podCIDRs) if err != nil { - return fmt.Errorf("failed to parse podCIDRs : %w", err) + return errors.Wrapf(err, "failed to parse podCIDRs") } // Get and parse serviceCIDRs from env serviceCIDRs, err := configuration.ServiceCIDRs() if err != nil { - return fmt.Errorf("failed to get serviceCIDRs from env : %w", err) + return errors.Wrapf(err, "failed to get serviceCIDRs from env") } serviceCIDRsV4, serviceCIDRsV6, err := utils.ParseCIDRs(serviceCIDRs) if err != nil { - return fmt.Errorf("failed to parse serviceCIDRs : %w", err) + return errors.Wrapf(err, "failed to parse serviceCIDRs") } // Check if the podIPInfo is IPv4 or IPv6 if net.ParseIP(podIPInfo.PodIPConfig.IPAddress).To4() != nil { diff --git a/cns/middlewares/swiftV2_test.go b/cns/middlewares/swiftV2_test.go index 404bdecd74..9de8bbe66a 100644 --- a/cns/middlewares/swiftV2_test.go +++ b/cns/middlewares/swiftV2_test.go @@ -98,7 +98,7 @@ func TestGetSWIFTv2IPConfigFailure(t *testing.T) { // Pod's MTPNC doesn't exist in cache test _, err := middleware.GetIPConfig(context.TODO(), testPod2Info) - assert.Error(t, err, "failed to get pod's mtpnc from cache : mtpnc not found") + assert.ErrorContains(t, err, mock.ErrMTPNCNotFound.Error()) // Pod's MTPNC is not ready test _, err = middleware.GetIPConfig(context.TODO(), testPod3Info) From d57e6b32eea9d00f8721f93776f9098ce2f78379 Mon Sep 17 00:00:00 2001 From: Quang Nguyen Date: Mon, 30 Oct 2023 13:41:29 -0400 Subject: [PATCH 83/85] fix: add mtpnc status check in validator + use netip package --- cns/azure-cns-swiftv2.yaml | 228 ---------------------------- cns/azure-cns.yaml | 108 ++++++++++--- cns/middlewares/mock/mockClient.go | 19 ++- cns/middlewares/mock/mockSWIFTv2.go | 18 ++- cns/middlewares/swiftV2.go | 21 ++- cns/middlewares/swiftV2_test.go | 17 ++- cns/middlewares/utils/utils.go | 2 +- 7 files changed, 149 insertions(+), 264 deletions(-) delete mode 100644 cns/azure-cns-swiftv2.yaml diff --git a/cns/azure-cns-swiftv2.yaml b/cns/azure-cns-swiftv2.yaml deleted file mode 100644 index 0cd7700aab..0000000000 --- a/cns/azure-cns-swiftv2.yaml +++ /dev/null @@ -1,228 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: azure-cns - namespace: kube-system ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: acn-multitenancy-editor -labels: - addonmanager.kubernetes.io/mode: Reconcile -rules: - - apiGroups: ["acn.azure.com"] - resources: ["clustersubnetstates"] - verbs: ["get", "list", "watch"] - - apiGroups: ["multitenancy.acn.azure.com"] - resources: - ["multitenantpodnetworkconfigs", "podnetworkinstances", "podnetworks"] - verbs: ["get", "list", "watch", "patch", "update"] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: acn-multitenancy-editor-binding -subjects: - - kind: ServiceAccount - name: azure-cns - namespace: kube-system -roleRef: - kind: ClusterRole - name: acn-multitenancy-editor - apiGroup: rbac.authorization.k8s.io ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - namespace: kube-system - name: nodeNetConfigEditor -rules: - - apiGroups: ["acn.azure.com"] - resources: ["nodenetworkconfigs"] - verbs: ["get", "list", "watch", "patch", "update"] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: pod-reader-all-namespaces -rules: - - apiGroups: [""] - resources: ["pods"] - verbs: ["get", "watch", "list"] - - apiGroups: [""] - resources: ["nodes"] - verbs: ["get"] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: nodeNetConfigEditorRoleBinding - namespace: kube-system -subjects: - - kind: ServiceAccount - name: azure-cns - namespace: kube-system -roleRef: - kind: Role - name: nodeNetConfigEditor - apiGroup: rbac.authorization.k8s.io ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: pod-reader-all-namespaces-binding -subjects: - - kind: ServiceAccount - name: azure-cns - namespace: kube-system -roleRef: - kind: ClusterRole - name: pod-reader-all-namespaces - apiGroup: rbac.authorization.k8s.io ---- -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: azure-cns - namespace: kube-system - labels: - app: azure-cns -spec: - selector: - matchLabels: - k8s-app: azure-cns - template: - metadata: - labels: - k8s-app: azure-cns - annotations: - cluster-autoscaler.kubernetes.io/daemonset-pod: "true" - spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: kubernetes.azure.com/cluster - operator: Exists - - key: type - operator: NotIn - values: - - virtual-kubelet - - key: beta.kubernetes.io/os - operator: In - values: - - linux - priorityClassName: system-node-critical - tolerations: - - key: CriticalAddonsOnly - operator: Exists - - operator: "Exists" - effect: NoExecute - - operator: "Exists" - effect: NoSchedule - containers: - - name: cns-container - image: mcr.microsoft.com/containernetworking/azure-cns:v1.4.7 - imagePullPolicy: IfNotPresent - args: - ["-c", "tcp://$(CNSIpAddress):$(CNSPort)", "-t", "$(CNSLogTarget)"] - volumeMounts: - - name: log - mountPath: /var/log - - name: cns-state - mountPath: /var/lib/azure-network - - name: azure-endpoints - mountPath: /var/run/azure-cns/ - - name: cns-config - mountPath: /etc/azure-cns - - name: cni-bin - mountPath: /opt/cni/bin - - name: azure-vnet - mountPath: /var/run/azure-vnet - - name: legacy-cni-state - mountPath: /var/run/azure-vnet.json - ports: - - containerPort: 10090 - env: - - name: CNSIpAddress - value: "127.0.0.1" - - name: CNSPort - value: "10090" - - name: CNSLogTarget - value: "stdoutfile" - - name: CNS_CONFIGURATION_PATH - value: /etc/azure-cns/cns_config.json - - name: NODENAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - - name: POD_CIDRs - value: 10.0.0.0/24 # example value - - name: SERVICE_CIDRs - value: 10.0.0.0/25 # example value - - name: NODE_CIDRs - value: 198.19.0.128/25 # example value - hostNetwork: true - volumes: - - name: azure-endpoints - hostPath: - path: /var/run/azure-cns/ - type: DirectoryOrCreate - - name: log - hostPath: - path: /var/log - type: Directory - - name: cns-state - hostPath: - path: /var/lib/azure-network - type: DirectoryOrCreate - - name: cni-bin - hostPath: - path: /opt/cni/bin - type: Directory - - name: azure-vnet - hostPath: - path: /var/run/azure-vnet - type: DirectoryOrCreate - - name: legacy-cni-state - hostPath: - path: /var/run/azure-vnet.json - type: FileOrCreate - - name: cns-config - configMap: - name: cns-config - serviceAccountName: azure-cns ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: cns-config - namespace: kube-system -data: - cns_config.json: | - { - "TelemetrySettings": { - "TelemetryBatchSizeBytes": 16384, - "TelemetryBatchIntervalInSecs": 15, - "RefreshIntervalInSecs": 15, - "DisableAll": false, - "HeartBeatIntervalInMins": 30, - "DebugMode": false, - "SnapshotIntervalInMins": 60 - }, - "ManagedSettings": { - "PrivateEndpoint": "", - "InfrastructureNetworkID": "", - "NodeID": "", - "NodeSyncIntervalInSeconds": 30 - }, - "ChannelMode": "CRD", - "InitializeFromCNI": true, - "ManageEndpointState": false, - "ProgramSNATIPTables" : false - } - -# Toggle ManageEndpointState and ProgramSNATIPTables to true for delegated IPAM use case. diff --git a/cns/azure-cns.yaml b/cns/azure-cns.yaml index 12c3cd249e..61b32decbd 100644 --- a/cns/azure-cns.yaml +++ b/cns/azure-cns.yaml @@ -1,12 +1,68 @@ apiVersion: v1 +kind: ServiceAccount +metadata: + name: azure-cns + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: acn-multitenancy-editor +labels: + addonmanager.kubernetes.io/mode: Reconcile +rules: + - apiGroups: ["acn.azure.com"] + resources: ["clustersubnetstates"] + verbs: ["get", "list", "watch"] + - apiGroups: ["multitenancy.acn.azure.com"] + resources: + ["multitenantpodnetworkconfigs", "podnetworkinstances", "podnetworks"] + verbs: ["get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: acn-multitenancy-editor-binding +subjects: + - kind: ServiceAccount + name: azure-cns + namespace: kube-system +roleRef: + kind: ClusterRole + name: acn-multitenancy-editor + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + namespace: kube-system + name: nodeNetConfigEditor +rules: + - apiGroups: ["acn.azure.com"] + resources: ["nodenetworkconfigs"] + verbs: ["get", "list", "watch", "patch", "update"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: pod-reader-all-namespaces +rules: + - apiGroups: [""] + resources: ["pods"] + verbs: ["get", "watch", "list"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get"] +--- +apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: name: nodeNetConfigEditorRoleBinding namespace: kube-system subjects: -- kind: ServiceAccount - name: azure-cns - namespace: kube-system + - kind: ServiceAccount + name: azure-cns + namespace: kube-system roleRef: kind: Role name: nodeNetConfigEditor @@ -17,9 +73,9 @@ kind: ClusterRoleBinding metadata: name: pod-reader-all-namespaces-binding subjects: -- kind: ServiceAccount - name: azure-cns - namespace: kube-system + - kind: ServiceAccount + name: azure-cns + namespace: kube-system roleRef: kind: ClusterRole name: pod-reader-all-namespaces @@ -47,17 +103,17 @@ spec: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - - matchExpressions: - - key: kubernetes.azure.com/cluster - operator: Exists - - key: type - operator: NotIn - values: - - virtual-kubelet - - key: beta.kubernetes.io/os - operator: In - values: - - linux + - matchExpressions: + - key: kubernetes.azure.com/cluster + operator: Exists + - key: type + operator: NotIn + values: + - virtual-kubelet + - key: beta.kubernetes.io/os + operator: In + values: + - linux priorityClassName: system-node-critical tolerations: - key: CriticalAddonsOnly @@ -70,7 +126,8 @@ spec: - name: cns-container image: mcr.microsoft.com/containernetworking/azure-cns:v1.4.7 imagePullPolicy: IfNotPresent - args: [ "-c", "tcp://$(CNSIpAddress):$(CNSPort)", "-t", "$(CNSLogTarget)"] + args: + ["-c", "tcp://$(CNSIpAddress):$(CNSPort)", "-t", "$(CNSLogTarget)"] volumeMounts: - name: log mountPath: /var/log @@ -99,9 +156,16 @@ spec: value: /etc/azure-cns/cns_config.json - name: NODENAME valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + - name: POD_CIDRs + value: 10.0.0.0/16 + - name: SERVICE_CIDRs + value: 10.1.0.0/16 + - name: NODE_CIDRs + value: 192.168.0.0/16 + # POD_CIDRs, SERVICE_CIDRs and NODE_CIDRs are set for SWIFT v2 scenario. hostNetwork: true volumes: - name: azure-endpoints @@ -161,4 +225,4 @@ data: "ManageEndpointState": false, "ProgramSNATIPTables" : false } -# Toggle ManageEndpointState and ProgramSNATIPTables to true for delegated IPAM use case. +# Toggle ManageEndpointState and ProgramSNATIPTables to true for delegated IPAM use case. \ No newline at end of file diff --git a/cns/middlewares/mock/mockClient.go b/cns/middlewares/mock/mockClient.go index bf6a14deeb..0bb8be743b 100644 --- a/cns/middlewares/mock/mockClient.go +++ b/cns/middlewares/mock/mockClient.go @@ -24,23 +24,34 @@ type Client struct { // NewClient returns a new MockClient. func NewClient() *Client { + const podNetwork = "azure" + testPod1 := v1.Pod{} testPod1.Labels = make(map[string]string) - testPod1.Labels[configuration.LabelPodSwiftV2] = "true" + testPod1.Labels[configuration.LabelPodSwiftV2] = podNetwork + + testPod3 := v1.Pod{} + testPod3.Labels = make(map[string]string) + testPod3.Labels[configuration.LabelPodSwiftV2] = podNetwork + + testPod4 := v1.Pod{} + testPod4.Labels = make(map[string]string) + testPod4.Labels[configuration.LabelPodSwiftV2] = podNetwork testMTPNC1 := v1alpha1.MultitenantPodNetworkConfig{} + testMTPNC1.Status.PrimaryIP = "192.168.0.1/32" testMTPNC1.Status.MacAddress = "00:00:00:00:00:00" testMTPNC1.Status.GatewayIP = "10.0.0.1" testMTPNC1.Status.NCID = "testncid" - testMTPNC3 := v1alpha1.MultitenantPodNetworkConfig{} + testMTPNC4 := v1alpha1.MultitenantPodNetworkConfig{} return &Client{ - mtPodCache: map[string]*v1.Pod{"testpod1namespace/testpod1": &testPod1}, + mtPodCache: map[string]*v1.Pod{"testpod1namespace/testpod1": &testPod1, "testpod3namespace/testpod3": &testPod3, "testpod4namespace/testpod4": &testPod4}, mtpncCache: map[string]*v1alpha1.MultitenantPodNetworkConfig{ "testpod1namespace/testpod1": &testMTPNC1, - "testpod3namespace/testpod3": &testMTPNC3, + "testpod4namespace/testpod4": &testMTPNC4, }, } } diff --git a/cns/middlewares/mock/mockSWIFTv2.go b/cns/middlewares/mock/mockSWIFTv2.go index 9d35fdf2e6..18347b7945 100644 --- a/cns/middlewares/mock/mockSWIFTv2.go +++ b/cns/middlewares/mock/mockSWIFTv2.go @@ -18,6 +18,7 @@ import ( var ( errMTPNCNotReady = errors.New("mtpnc is not ready") + errFailedToGetMTPNC = errors.New("failed to get mtpnc") errFailedToGetPod = errors.New("failed to get pod") errInvalidSWIFTv2NICType = errors.New("invalid NIC type for SWIFT v2 scenario") ) @@ -90,6 +91,17 @@ func (m *SWIFTv2Middleware) ValidateIPConfigsRequest(_ context.Context, req *cns // check the pod labels for Swift V2, enrich the request with the multitenant flag. if _, ok := pod.Labels[configuration.LabelPodSwiftV2]; ok { req.SecondaryInterfacesExist = true + // Check if the MTPNC CRD exists for the pod, if not, return error + mtpncNamespacedName := k8types.NamespacedName{Namespace: podInfo.Namespace(), Name: podInfo.Name()} + mtpnc, ok := m.mtpncState[mtpncNamespacedName.String()] + if !ok { + return types.UnexpectedError, errFailedToGetMTPNC.Error() + } + + // Check if the MTPNC CRD is ready. If one of the fields is empty, return error + if mtpnc.Status.PrimaryIP == "" || mtpnc.Status.MacAddress == "" || mtpnc.Status.NCID == "" || mtpnc.Status.GatewayIP == "" { + return types.UnexpectedError, errMTPNCNotReady.Error() + } } return types.Success, "" } @@ -101,13 +113,9 @@ func (m *SWIFTv2Middleware) GetIPConfig(_ context.Context, podInfo cns.PodInfo) mtpncNamespacedName := k8types.NamespacedName{Namespace: podInfo.Namespace(), Name: podInfo.Name()} mtpnc, ok := m.mtpncState[mtpncNamespacedName.String()] if !ok { - return cns.PodIpInfo{}, errFailedToGetPod + return cns.PodIpInfo{}, errFailedToGetMTPNC } - // Check if the MTPNC CRD is ready. If one of the fields is empty, return error - if mtpnc.Status.PrimaryIP == "" || mtpnc.Status.MacAddress == "" || mtpnc.Status.NCID == "" || mtpnc.Status.GatewayIP == "" { - return cns.PodIpInfo{}, errMTPNCNotReady - } podIPInfo := cns.PodIpInfo{} podIPInfo.PodIPConfig = cns.IPSubnet{ IPAddress: mtpnc.Status.PrimaryIP, diff --git a/cns/middlewares/swiftV2.go b/cns/middlewares/swiftV2.go index 3eb1ef79ac..8825c99215 100644 --- a/cns/middlewares/swiftV2.go +++ b/cns/middlewares/swiftV2.go @@ -2,7 +2,7 @@ package middlewares import ( "context" - "net" + "fmt" "net/netip" "github.com/Azure/azure-container-networking/cns" @@ -50,9 +50,19 @@ func (m *SWIFTv2Middleware) ValidateIPConfigsRequest(ctx context.Context, req *c return types.UnexpectedError, errBuf.Error() } - // check the pod labels for Swift V2, set the request's SecondaryInterfaceSet flag to true. + // check the pod labels for Swift V2, set the request's SecondaryInterfaceSet flag to true and check if its MTPNC CRD is ready if _, ok := pod.Labels[configuration.LabelPodSwiftV2]; ok { req.SecondaryInterfacesExist = true + // Check if the MTPNC CRD exists for the pod, if not, return error + mtpnc := v1alpha1.MultitenantPodNetworkConfig{} + mtpncNamespacedName := k8stypes.NamespacedName{Namespace: podInfo.Namespace(), Name: podInfo.Name()} + if err := m.Cli.Get(ctx, mtpncNamespacedName, &mtpnc); err != nil { + return types.UnexpectedError, fmt.Errorf("failed to get pod's mtpnc from cache : %w", err).Error() + } + // Check if the MTPNC CRD is ready. If one of the fields is empty, return error + if mtpnc.Status.PrimaryIP == "" || mtpnc.Status.MacAddress == "" || mtpnc.Status.NCID == "" || mtpnc.Status.GatewayIP == "" { + return types.UnexpectedError, errMTPNCNotReady.Error() + } } logger.Printf("[SWIFTv2Middleware] pod %s has secondary interface : %v", podInfo.Name(), req.SecondaryInterfacesExist) return types.Success, "" @@ -72,6 +82,7 @@ func (m *SWIFTv2Middleware) GetIPConfig(ctx context.Context, podInfo cns.PodInfo return cns.PodIpInfo{}, errMTPNCNotReady } logger.Printf("[SWIFTv2Middleware] mtpnc for pod %s is : %+v", podInfo.Name(), mtpnc) + // Parse MTPNC primaryIP to get the IP address and prefix length p, err := netip.ParsePrefix(mtpnc.Status.PrimaryIP) if err != nil { @@ -139,7 +150,11 @@ func (m *SWIFTv2Middleware) SetRoutes(podIPInfo *cns.PodIpInfo) error { return errors.Wrapf(err, "failed to parse serviceCIDRs") } // Check if the podIPInfo is IPv4 or IPv6 - if net.ParseIP(podIPInfo.PodIPConfig.IPAddress).To4() != nil { + ip, err := netip.ParseAddr(podIPInfo.PodIPConfig.IPAddress) + if err != nil { + return errors.Wrapf(err, "failed to parse podIPConfig IP address %s", podIPInfo.PodIPConfig.IPAddress) + } + if ip.Is4() { // routes for IPv4 podCIDR traffic for _, podCIDRv4 := range podCIDRsV4 { podCIDRv4Route := cns.Route{ diff --git a/cns/middlewares/swiftV2_test.go b/cns/middlewares/swiftV2_test.go index 9de8bbe66a..74a2daf246 100644 --- a/cns/middlewares/swiftV2_test.go +++ b/cns/middlewares/swiftV2_test.go @@ -22,6 +22,9 @@ var ( testPod3GUID = "718e04ac-5a13-4dce-84b3-040accaa9b41" testPod3Info = cns.NewPodInfo("718e04-eth0", testPod3GUID, "testpod3", "testpod3namespace") + + testPod4GUID = "b21e1ee1-fb7e-4e6d-8c68-22ee5049944e" + testPod4Info = cns.NewPodInfo("b21e1e-eth0", testPod4GUID, "testpod4", "testpod4namespace") ) func setEnvVar() { @@ -79,6 +82,18 @@ func TestValidateMultitenantIPConfigsRequestFailure(t *testing.T) { failReq.OrchestratorContext = b respCode, _ = middleware.ValidateIPConfigsRequest(context.TODO(), failReq) assert.Equal(t, respCode, types.UnexpectedError) + + // Failed to get MTPNC + b, _ = testPod3Info.OrchestratorContext() + failReq.OrchestratorContext = b + respCode, _ = middleware.ValidateIPConfigsRequest(context.TODO(), failReq) + assert.Equal(t, respCode, types.UnexpectedError) + + // MTPNC not ready + b, _ = testPod4Info.OrchestratorContext() + failReq.OrchestratorContext = b + respCode, _ = middleware.ValidateIPConfigsRequest(context.TODO(), failReq) + assert.Equal(t, respCode, types.UnexpectedError) } func TestGetSWIFTv2IPConfigSuccess(t *testing.T) { @@ -101,7 +116,7 @@ func TestGetSWIFTv2IPConfigFailure(t *testing.T) { assert.ErrorContains(t, err, mock.ErrMTPNCNotFound.Error()) // Pod's MTPNC is not ready test - _, err = middleware.GetIPConfig(context.TODO(), testPod3Info) + _, err = middleware.GetIPConfig(context.TODO(), testPod4Info) assert.Error(t, err, errMTPNCNotReady.Error()) } diff --git a/cns/middlewares/utils/utils.go b/cns/middlewares/utils/utils.go index 19766a2c39..2eca8551ea 100644 --- a/cns/middlewares/utils/utils.go +++ b/cns/middlewares/utils/utils.go @@ -6,7 +6,7 @@ import ( "strings" ) -// ParseCIDRs parses the semicolons separated CIDRs string and returns the IPv4 and IPv6 CIDRs. +// ParseCIDRs parses the comma separated list of CIDRs and returns the IPv4 and IPv6 CIDRs. func ParseCIDRs(cidrs string) (v4IPs, v6IPs []string, err error) { v4IPs = []string{} v6IPs = []string{} From 04f08fefa41628e2912052bfcbce20ec2da215f3 Mon Sep 17 00:00:00 2001 From: Quang Nguyen Date: Mon, 30 Oct 2023 13:58:55 -0400 Subject: [PATCH 84/85] fix: minor --- cns/middlewares/mock/mockSWIFTv2.go | 17 ++++++++++------- cns/middlewares/swiftV2.go | 4 ++-- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/cns/middlewares/mock/mockSWIFTv2.go b/cns/middlewares/mock/mockSWIFTv2.go index 18347b7945..11d1266f76 100644 --- a/cns/middlewares/mock/mockSWIFTv2.go +++ b/cns/middlewares/mock/mockSWIFTv2.go @@ -3,7 +3,7 @@ package mock import ( "context" "fmt" - "net" + "net/netip" "os" "github.com/Azure/azure-container-networking/cns" @@ -19,7 +19,6 @@ import ( var ( errMTPNCNotReady = errors.New("mtpnc is not ready") errFailedToGetMTPNC = errors.New("failed to get mtpnc") - errFailedToGetPod = errors.New("failed to get pod") errInvalidSWIFTv2NICType = errors.New("invalid NIC type for SWIFT v2 scenario") ) @@ -79,14 +78,14 @@ func (m *SWIFTv2Middleware) ValidateIPConfigsRequest(_ context.Context, req *cns // Retrieve the pod from the cluster podInfo, err := cns.UnmarshalPodInfo(req.OrchestratorContext) if err != nil { - errBuf := fmt.Sprintf("unmarshalling pod info from ipconfigs request %v failed with error %v", req, err) - return types.UnexpectedError, errBuf + errBuf := errors.Wrapf(err, "failed to unmarshalling pod info from ipconfigs request %+v", req) + return types.UnexpectedError, errBuf.Error() } podNamespacedName := k8types.NamespacedName{Namespace: podInfo.Namespace(), Name: podInfo.Name()} pod, ok := m.mtPodState[podNamespacedName.String()] if !ok { - errBuf := fmt.Sprintf("failed to get pod %v with error %v", podNamespacedName, err) - return types.UnexpectedError, errBuf + errBuf := errors.Wrapf(err, "failed to get pod %+v", podNamespacedName) + return types.UnexpectedError, errBuf.Error() } // check the pod labels for Swift V2, enrich the request with the multitenant flag. if _, ok := pod.Labels[configuration.LabelPodSwiftV2]; ok { @@ -167,7 +166,11 @@ func (m *SWIFTv2Middleware) SetRoutes(podIPInfo *cns.PodIpInfo) error { return errors.Wrapf(err, "failed to parse serviceCIDRs") } // Check if the podIPInfo is IPv4 or IPv6 - if net.ParseIP(podIPInfo.PodIPConfig.IPAddress).To4() != nil { + ip, err := netip.ParseAddr(podIPInfo.PodIPConfig.IPAddress) + if err != nil { + return errors.Wrapf(err, "failed to parse podIPConfig IP address %s", podIPInfo.PodIPConfig.IPAddress) + } + if ip.Is4() { // routes for IPv4 podCIDR traffic for _, podCIDRv4 := range podCIDRsV4 { podCIDRv4Route := cns.Route{ diff --git a/cns/middlewares/swiftV2.go b/cns/middlewares/swiftV2.go index 8825c99215..7326ccc6bd 100644 --- a/cns/middlewares/swiftV2.go +++ b/cns/middlewares/swiftV2.go @@ -39,14 +39,14 @@ func (m *SWIFTv2Middleware) ValidateIPConfigsRequest(ctx context.Context, req *c // Retrieve the pod from the cluster podInfo, err := cns.UnmarshalPodInfo(req.OrchestratorContext) if err != nil { - errBuf := errors.Wrapf(err, "failed to unmarshalling pod info from ipconfigs request %v", req) + errBuf := errors.Wrapf(err, "failed to unmarshalling pod info from ipconfigs request %+v", req) return types.UnexpectedError, errBuf.Error() } logger.Printf("[SWIFTv2Middleware] validate ipconfigs request for pod %s", podInfo.Name()) podNamespacedName := k8stypes.NamespacedName{Namespace: podInfo.Namespace(), Name: podInfo.Name()} pod := v1.Pod{} if err := m.Cli.Get(ctx, podNamespacedName, &pod); err != nil { - errBuf := errors.Wrapf(err, "failed to get pod %v", podNamespacedName) + errBuf := errors.Wrapf(err, "failed to get pod %+v", podNamespacedName) return types.UnexpectedError, errBuf.Error() } From 113021ded50c06daa464a0264b8a0146981c6c2b Mon Sep 17 00:00:00 2001 From: Quang Nguyen Date: Tue, 31 Oct 2023 11:18:02 -0400 Subject: [PATCH 85/85] revert: azure-cns.yaml --- cns/azure-cns.yaml | 96 +++++++++++++++------------------------------- 1 file changed, 30 insertions(+), 66 deletions(-) diff --git a/cns/azure-cns.yaml b/cns/azure-cns.yaml index 61b32decbd..260bb775c1 100644 --- a/cns/azure-cns.yaml +++ b/cns/azure-cns.yaml @@ -5,54 +5,26 @@ metadata: namespace: kube-system --- apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: acn-multitenancy-editor -labels: - addonmanager.kubernetes.io/mode: Reconcile -rules: - - apiGroups: ["acn.azure.com"] - resources: ["clustersubnetstates"] - verbs: ["get", "list", "watch"] - - apiGroups: ["multitenancy.acn.azure.com"] - resources: - ["multitenantpodnetworkconfigs", "podnetworkinstances", "podnetworks"] - verbs: ["get", "list", "watch"] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: acn-multitenancy-editor-binding -subjects: - - kind: ServiceAccount - name: azure-cns - namespace: kube-system -roleRef: - kind: ClusterRole - name: acn-multitenancy-editor - apiGroup: rbac.authorization.k8s.io ---- -apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: namespace: kube-system name: nodeNetConfigEditor rules: - - apiGroups: ["acn.azure.com"] - resources: ["nodenetworkconfigs"] - verbs: ["get", "list", "watch", "patch", "update"] +- apiGroups: ["acn.azure.com"] + resources: ["nodenetworkconfigs"] + verbs: ["get", "list", "watch", "patch", "update"] --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: pod-reader-all-namespaces rules: - - apiGroups: [""] - resources: ["pods"] - verbs: ["get", "watch", "list"] - - apiGroups: [""] - resources: ["nodes"] - verbs: ["get"] +- apiGroups: [""] + resources: ["pods"] + verbs: ["get", "watch", "list"] +- apiGroups: [""] + resources: ["nodes"] + verbs: ["get"] --- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding @@ -60,9 +32,9 @@ metadata: name: nodeNetConfigEditorRoleBinding namespace: kube-system subjects: - - kind: ServiceAccount - name: azure-cns - namespace: kube-system +- kind: ServiceAccount + name: azure-cns + namespace: kube-system roleRef: kind: Role name: nodeNetConfigEditor @@ -73,9 +45,9 @@ kind: ClusterRoleBinding metadata: name: pod-reader-all-namespaces-binding subjects: - - kind: ServiceAccount - name: azure-cns - namespace: kube-system +- kind: ServiceAccount + name: azure-cns + namespace: kube-system roleRef: kind: ClusterRole name: pod-reader-all-namespaces @@ -103,17 +75,17 @@ spec: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - - matchExpressions: - - key: kubernetes.azure.com/cluster - operator: Exists - - key: type - operator: NotIn - values: - - virtual-kubelet - - key: beta.kubernetes.io/os - operator: In - values: - - linux + - matchExpressions: + - key: kubernetes.azure.com/cluster + operator: Exists + - key: type + operator: NotIn + values: + - virtual-kubelet + - key: beta.kubernetes.io/os + operator: In + values: + - linux priorityClassName: system-node-critical tolerations: - key: CriticalAddonsOnly @@ -126,8 +98,7 @@ spec: - name: cns-container image: mcr.microsoft.com/containernetworking/azure-cns:v1.4.7 imagePullPolicy: IfNotPresent - args: - ["-c", "tcp://$(CNSIpAddress):$(CNSPort)", "-t", "$(CNSLogTarget)"] + args: [ "-c", "tcp://$(CNSIpAddress):$(CNSPort)", "-t", "$(CNSLogTarget)"] volumeMounts: - name: log mountPath: /var/log @@ -156,16 +127,9 @@ spec: value: /etc/azure-cns/cns_config.json - name: NODENAME valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: spec.nodeName - - name: POD_CIDRs - value: 10.0.0.0/16 - - name: SERVICE_CIDRs - value: 10.1.0.0/16 - - name: NODE_CIDRs - value: 192.168.0.0/16 - # POD_CIDRs, SERVICE_CIDRs and NODE_CIDRs are set for SWIFT v2 scenario. + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName hostNetwork: true volumes: - name: azure-endpoints