diff --git a/Makefile b/Makefile index a2610e254e..6e821d8e2d 100644 --- a/Makefile +++ b/Makefile @@ -49,6 +49,8 @@ CNSFILES = \ $(wildcard cns/routes/*.go) \ $(wildcard cns/service/*.go) \ $(wildcard cns/networkcontainers/*.go) \ + $(wildcard cns/requestcontroller/*.go) \ + $(wildcard cns/requestcontroller/kubecontroller/*.go) \ $(COREFILES) \ $(CNMFILES) diff --git a/cns/NetworkContainerContract.go b/cns/NetworkContainerContract.go index 6c0aaa7d75..c2cbe60f00 100644 --- a/cns/NetworkContainerContract.go +++ b/cns/NetworkContainerContract.go @@ -44,6 +44,8 @@ const ( Batch = "Batch" DBforPostgreSQL = "DBforPostgreSQL" AzureFirstParty = "AzureFirstParty" + KubernetesCRD = "KubernetesCRD" + // TODO: Add OrchastratorType as CRD: https://msazure.visualstudio.com/One/_workitems/edit/7711872 ) // Encap Types @@ -98,6 +100,7 @@ type KubernetesPodInfo struct { } // GetOrchestratorContext will return the orchestratorcontext as a string +// TODO - should use a hashed name or can this be PODUid? func (podinfo *KubernetesPodInfo) GetOrchestratorContextKey() string { return podinfo.PodName + ":" + podinfo.PodNamespace } @@ -115,16 +118,9 @@ type IPConfiguration struct { GatewayIPAddress string } +// SecondaryIPConfig contains IP info of SecondaryIP type SecondaryIPConfig struct { - IPConfig IPSubnet -} - -type ContainerIPConfigState struct { - IPConfig IPSubnet - ID string //uuid - NCID string - State string - OrchestratorContext json.RawMessage + IPSubnet IPSubnet } // IPSubnet contains ip subnet. diff --git a/cns/cnsclient/cnsclient_test.go b/cns/cnsclient/cnsclient_test.go index 99a7f2aa20..3b3e2137ef 100644 --- a/cns/cnsclient/cnsclient_test.go +++ b/cns/cnsclient/cnsclient_test.go @@ -1,35 +1,68 @@ package cnsclient import ( + "bytes" + "encoding/json" + "fmt" + "io/ioutil" "net" + "net/http" + "os" + "reflect" "strconv" + "testing" "github.com/Azure/azure-container-networking/cns" + "github.com/Azure/azure-container-networking/cns/common" + "github.com/Azure/azure-container-networking/cns/logger" + "github.com/Azure/azure-container-networking/cns/restserver" + "github.com/Azure/azure-container-networking/log" + "github.com/google/uuid" ) var ( - testNCID = "06867cf3-332d-409d-8819-ed70d2c116b0" + svc *restserver.HTTPRestService +) - testIP1 = "10.0.0.1" - testPod1GUID = "898fb8f1-f93e-4c96-9c31-6b89098949a3" - testPod1Info = cns.KubernetesPodInfo{ - PodName: "testpod1", - PodNamespace: "testpod1namespace", - } +const ( + primaryIp = "10.0.0.5" + gatewayIp = "10.0.0.1" + dockerContainerType = cns.Docker ) -// func addTestStateToRestServer(svc *restserver.HTTPRestService) { -// // set state as already allocated -// state1, _ := restserver.NewPodStateWithOrchestratorContext(testIP1, 24, testPod1GUID, testNCID, cns.Available, testPod1Info) -// ipconfigs := map[string]cns.SecondaryIPConfig{ -// state1.ID: state1, -// } -// nc := cns.CreateNetworkContainerRequest{ -// SecondaryIPConfigs: ipconfigs, -// } +func addTestStateToRestServer(t *testing.T, secondaryIps []string) { + var ipConfig cns.IPConfiguration + ipConfig.DNSServers = []string{"8.8.8.8", "8.8.4.4"} + ipConfig.GatewayIPAddress = gatewayIp + var ipSubnet cns.IPSubnet + ipSubnet.IPAddress = primaryIp + ipSubnet.PrefixLength = 32 + ipConfig.IPSubnet = ipSubnet + secondaryIPConfigs := make(map[string]cns.SecondaryIPConfig) + + for _, secIpAddress := range secondaryIps { + secIpConfig := cns.SecondaryIPConfig{ + IPSubnet: cns.IPSubnet{ + IPAddress: secIpAddress, + PrefixLength: 32, + }, + } + ipId := uuid.New() + secondaryIPConfigs[ipId.String()] = secIpConfig + } -// svc.CreateOrUpdateNetworkContainerWithSecondaryIPConfigs(nc) -// } + req := cns.CreateNetworkContainerRequest{ + NetworkContainerType: dockerContainerType, + NetworkContainerid: "testNcId1", + IPConfiguration: ipConfig, + SecondaryIPConfigs: secondaryIPConfigs, + } + + returnCode := svc.CreateOrUpdateNetworkContainerInternal(req) + if returnCode != 0 { + t.Fatalf("Failed to createNetworkContainerRequest, req: %+v, err: %d", req, returnCode) + } +} func getIPConfigFromGetNetworkContainerResponse(resp *cns.GetIPConfigResponse) (net.IPNet, error) { var ( @@ -52,11 +85,10 @@ func getIPConfigFromGetNetworkContainerResponse(resp *cns.GetIPConfigResponse) ( return resultIPnet, err } -/* func TestMain(m *testing.M) { var ( info = &cns.SetOrchestratorTypeRequest{ - OrchestratorType: cns.Kubernetes} + OrchestratorType: cns.KubernetesCRD} body bytes.Buffer res *http.Response ) @@ -80,15 +112,13 @@ func TestMain(m *testing.M) { config := common.ServiceConfig{} httpRestService, err := restserver.NewHTTPRestService(&config) - svc := httpRestService.(*restserver.HTTPRestService) + svc = httpRestService.(*restserver.HTTPRestService) svc.Name = "cns-test-server" if err != nil { logger.Errorf("Failed to create CNS object, err:%v.\n", err) return } - //addTestStateToRestServer(svc) - if httpRestService != nil { err = httpRestService.Start(&config) if err != nil { @@ -111,31 +141,37 @@ func TestMain(m *testing.M) { } fmt.Println(res) - m.Run() + exitCode := m.Run() + os.Exit(exitCode) } func TestCNSClientRequestAndRelease(t *testing.T) { podName := "testpodname" podNamespace := "testpodnamespace" - ip := net.ParseIP("10.0.0.1") - _, ipnet, _ := net.ParseCIDR("10.0.0.1/24") + desiredIpAddress := "10.0.0.5" + ip := net.ParseIP(desiredIpAddress) + _, ipnet, _ := net.ParseCIDR("10.0.0.5/32") desired := net.IPNet{ IP: ip, Mask: ipnet.Mask, } + secondaryIps := make([]string, 0) + secondaryIps = append(secondaryIps, desiredIpAddress) cnsClient, _ := InitCnsClient("") + addTestStateToRestServer(t, secondaryIps) + podInfo := cns.KubernetesPodInfo{PodName: podName, PodNamespace: podNamespace} orchestratorContext, err := json.Marshal(podInfo) if err != nil { t.Fatal(err) } - // no IP reservation found with that context, expect fail + // no IP reservation found with that context, expect no failure. err = cnsClient.ReleaseIPAddress(orchestratorContext) - if err == nil { - t.Fatalf("Expected failure to release when no IP reservation found with context: %+v", err) + if err != nil { + t.Fatalf("Release ip idempotent call failed: %+v", err) } // request IP address @@ -156,4 +192,3 @@ func TestCNSClientRequestAndRelease(t *testing.T) { t.Fatalf("Expected to not fail when releasing IP reservation found with context: %+v", err) } } -*/ diff --git a/cns/cnsclient/httpapi/client.go b/cns/cnsclient/httpapi/client.go index f664fc5ea4..396a1f7582 100644 --- a/cns/cnsclient/httpapi/client.go +++ b/cns/cnsclient/httpapi/client.go @@ -1,6 +1,8 @@ package httpapi import ( + "fmt" + "github.com/Azure/azure-container-networking/cns" "github.com/Azure/azure-container-networking/cns/restserver" ) @@ -12,24 +14,13 @@ type Client struct { // CreateOrUpdateNC updates cns state func (client *Client) CreateOrUpdateNC(ncRequest *cns.CreateNetworkContainerRequest) error { - // var ( - // ipConfigsToAdd []*cns.ContainerIPConfigState - // ) - - // //Lock to read ipconfigs - // client.RestService.Lock() + returnCode := client.RestService.CreateOrUpdateNetworkContainerInternal(*ncRequest) - // //Only add ipconfigs that don't exist in cns state already - // for _, ipConfig := range ipConfigs { - // if _, ok := client.RestService.PodIPConfigState[ipConfig.ID]; !ok { - // ipConfig.State = cns.Available - // ipConfigsToAdd = append(ipConfigsToAdd, ipConfig) - // } - // } + if returnCode != 0 { + return fmt.Errorf("Failed to Create NC request: %+v, errorCode: %d", *ncRequest, returnCode) + } - // client.RestService.Unlock() - // leave empty - return nil //client.RestService.AddIPConfigsToState(ipConfigsToAdd) + return nil } // InitCNSState initializes cns state diff --git a/cns/logger/log.go b/cns/logger/log.go index 450551e46e..bfccd9b3d5 100644 --- a/cns/logger/log.go +++ b/cns/logger/log.go @@ -84,8 +84,9 @@ func sendTraceInternal(msg string) { Log.th.TrackLog(report) } +// also logs in the AI telemetry func Printf(format string, args ...interface{}) { - Log.logger.Printf(format, args...) + Log.logger.Logf(format, args...) if Log.th == nil || Log.DisableTraceLogging { return diff --git a/cns/restserver/api.go b/cns/restserver/api.go index 7c8692c373..6fb4ecbfe5 100644 --- a/cns/restserver/api.go +++ b/cns/restserver/api.go @@ -726,6 +726,8 @@ func (service *HTTPRestService) setOrchestratorType(w http.ResponseWriter, r *ht fallthrough case cns.Kubernetes: fallthrough + case cns.KubernetesCRD: + fallthrough case cns.WebApps: fallthrough case cns.Batch: diff --git a/cns/restserver/api_test.go b/cns/restserver/api_test.go index 375bc8b201..29eb4013ce 100644 --- a/cns/restserver/api_test.go +++ b/cns/restserver/api_test.go @@ -51,6 +51,7 @@ type xmlDocument struct { var ( service HTTPService + svc *HTTPRestService mux *http.ServeMux hostQueryForProgrammedVersionResponse = `{"httpStatusCode":"200","networkContainerId":"eab2470f-test-test-test-b3cd316979d5","version":"1"}` hostQueryResponse = xmlDocument{ @@ -104,8 +105,7 @@ func TestMain(m *testing.M) { fmt.Printf("Failed to create CNS object %v\n", err) os.Exit(1) } - - svc := service.(*HTTPRestService) + svc = service.(*HTTPRestService) svc.Name = "cns-test-server" if err != nil { logger.Errorf("Failed to create CNS object, err:%v.\n", err) diff --git a/cns/restserver/const.go b/cns/restserver/const.go index 3534758072..518a83cafd 100644 --- a/cns/restserver/const.go +++ b/cns/restserver/const.go @@ -27,6 +27,10 @@ const ( NetworkJoinFailed = 24 NetworkContainerPublishFailed = 25 NetworkContainerUnpublishFailed = 26 + InvalidPrimaryIPConfig = 27 + PrimaryCANotSame = 28 + InconsistentIPConfigState = 29 + InvalidSecondaryIPConfig = 30 UnexpectedError = 99 ) diff --git a/cns/restserver/internalapi.go b/cns/restserver/internalapi.go index fab353c370..293ad3a4ff 100644 --- a/cns/restserver/internalapi.go +++ b/cns/restserver/internalapi.go @@ -3,6 +3,13 @@ package restserver +import ( + "reflect" + + "github.com/Azure/azure-container-networking/cns" + "github.com/Azure/azure-container-networking/cns/logger" +) + // This file contains the internal functions called by either HTTP APIs (api.go) or // internal APIs (definde in internalapi.go). // This will be used internally (say by RequestController in case of AKS) @@ -14,3 +21,58 @@ func (service *HTTPRestService) GetPartitionKey() (dncPartitionKey string) { service.RUnlock() return } + +// This API will be called by CNS RequestController on CRD update. +func (service *HTTPRestService) CreateOrUpdateNetworkContainerInternal(req cns.CreateNetworkContainerRequest) int { + if req.NetworkContainerid == "" { + logger.Errorf("[Azure CNS] Error. NetworkContainerid is empty") + return NetworkContainerNotSpecified + } + + // For now only RequestController uses this API which will be initialized only for AKS scenario. + // Validate ContainerType is set as Docker + if service.state.OrchestratorType != cns.KubernetesCRD { + logger.Errorf("[Azure CNS] Error. Unsupported OrchestratorType: %s", service.state.OrchestratorType) + return UnsupportedOrchestratorType + } + + // Validate PrimaryCA must never be empty + err := validateIPSubnet(req.IPConfiguration.IPSubnet) + if err != nil { + logger.Errorf("[Azure CNS] Error. PrimaryCA is invalid, NC Req: %v", req) + return InvalidPrimaryIPConfig + } + + // Validate SecondaryIPConfig + for ipId, secIpconfig := range req.SecondaryIPConfigs { + // Validate Ipconfig + err := validateIPSubnet(secIpconfig.IPSubnet) + if err != nil { + logger.Errorf("[Azure CNS] Error. SecondaryIpConfig, Id:%s is invalid, SecondaryIPConfig: %v, ncId: %s", ipId, secIpconfig, req.NetworkContainerid) + return InvalidSecondaryIPConfig + } + } + + // Validate if state exists already + existingNCInfo, ok := service.getNetworkContainerDetails(req.NetworkContainerid) + + if ok { + existingReq := existingNCInfo.CreateNetworkContainerRequest + if reflect.DeepEqual(existingReq.IPConfiguration, req.IPConfiguration) != true { + logger.Errorf("[Azure CNS] Error. PrimaryCA is not same, NCId %s, old CA %s, new CA %s", req.NetworkContainerid, existingReq.PrimaryInterfaceIdentifier, req.PrimaryInterfaceIdentifier) + return PrimaryCANotSame + } + } + + // This will Create Or Update the NC state. + returnCode, returnMessage := service.saveNetworkContainerGoalState(req) + + // If the NC was created successfully, log NC snapshot. + if returnCode == 0 { + logNCSnapshot(req) + } else { + logger.Errorf(returnMessage) + } + + return returnCode +} diff --git a/cns/restserver/internalapi_test.go b/cns/restserver/internalapi_test.go new file mode 100644 index 0000000000..a0a28f2f29 --- /dev/null +++ b/cns/restserver/internalapi_test.go @@ -0,0 +1,162 @@ +// Copyright 2020 Microsoft. All rights reserved. +// MIT License + +package restserver + +import ( + "fmt" + "testing" + "strconv" + + "github.com/Azure/azure-container-networking/cns" + "github.com/google/uuid" +) + +const ( + primaryIp = "10.0.0.5" + gatewayIp = "10.0.0.1" + dockerContainerType = cns.Docker +) + +func TestCreateOrUpdateNetworkContainerInternal(t *testing.T) { + fmt.Println("Test: TestCreateOrUpdateNetworkContainerInternal") + + setEnv(t) + setOrchestratorTypeInternal(cns.KubernetesCRD) + + validateCreateOrUpdateNCInternal(t, 2) +} + +func setOrchestratorTypeInternal(orchestratorType string) { + fmt.Println("setOrchestratorTypeInternal") + svc.state.OrchestratorType = orchestratorType +} + +func validateCreateOrUpdateNCInternal(t *testing.T, secondaryIpCount int) { + secondaryIPConfigs := make(map[string]cns.SecondaryIPConfig) + + var startingIndex = 6 + for i := 0; i < secondaryIpCount; i++ { + ipaddress := "10.0.0." + strconv.Itoa(startingIndex) + secIpConfig := newSecondaryIPConfig(ipaddress, 32) + ipId := uuid.New() + secondaryIPConfigs[ipId.String()] = secIpConfig + startingIndex++ + } + + createAndValidateNCRequest(t, secondaryIPConfigs) + + // now Validate Update, add more secondaryIpConfig and it should handle the update + fmt.Println("Validate Scaleup") + for i := 0; i < secondaryIpCount; i++ { + ipaddress := "10.0.0." + strconv.Itoa(startingIndex) + secIpConfig := newSecondaryIPConfig(ipaddress, 32) + ipId := uuid.New() + secondaryIPConfigs[ipId.String()] = secIpConfig + startingIndex++ + } + + createAndValidateNCRequest(t, secondaryIPConfigs) + + // now Scale down, delete 3 ipaddresses from secondaryIpConfig req + fmt.Println("Validate Scale down") + var count = 0 + for ipid := range secondaryIPConfigs { + delete(secondaryIPConfigs, ipid) + count++ + + if count > secondaryIpCount { + break + } + } + + createAndValidateNCRequest(t, secondaryIPConfigs) + + // Cleanup all SecondaryIps + fmt.Println("Validate no SecondaryIpconfigs") + for ipid := range secondaryIPConfigs { + delete(secondaryIPConfigs, ipid) + } + + createAndValidateNCRequest(t, secondaryIPConfigs) +} + +func createAndValidateNCRequest(t *testing.T, secondaryIPConfigs map[string]cns.SecondaryIPConfig) { + req := generateNetworkContainerRequest(secondaryIPConfigs) + returnCode := svc.CreateOrUpdateNetworkContainerInternal(req) + if returnCode != 0 { + t.Fatalf("Failed to createNetworkContainerRequest, req: %+v, err: %d", req, returnCode) + } + validateNetworkRequest(t, req) +} + +// Validate the networkRequest is persisted. +func validateNetworkRequest(t *testing.T, req cns.CreateNetworkContainerRequest) { + containerStatus := svc.state.ContainerStatus[req.NetworkContainerid] + + if containerStatus.ID != req.NetworkContainerid { + t.Fatalf("Failed as NCId is not persisted, expected:%s, actual %s", req.NetworkContainerid, containerStatus.ID) + } + + actualReq := containerStatus.CreateNetworkContainerRequest + if actualReq.NetworkContainerType != req.NetworkContainerType { + t.Fatalf("Failed as ContainerTyper doesnt match, expected:%s, actual %s", req.NetworkContainerType, actualReq.NetworkContainerType) + } + + if actualReq.IPConfiguration.IPSubnet.IPAddress != req.IPConfiguration.IPSubnet.IPAddress { + t.Fatalf("Failed as Primary IPAddress doesnt match, expected:%s, actual %s", req.IPConfiguration.IPSubnet.IPAddress, actualReq.IPConfiguration.IPSubnet.IPAddress) + } + + // Validate Secondary ips are added in the PodMap + if len(svc.PodIPConfigState) != len(req.SecondaryIPConfigs) { + t.Fatalf("Failed as Secondary IP count doesnt match in PodIpConfig state, expected:%d, actual %d", len(req.SecondaryIPConfigs), len(svc.PodIPConfigState)) + } + + var alreadyValidated = make(map[string]string) + for ipid, ipStatus := range svc.PodIPConfigState { + if ipaddress, found := alreadyValidated[ipid]; !found { + if secondaryIpConfig, ok := req.SecondaryIPConfigs[ipid]; !ok { + t.Fatalf("PodIpConfigState has stale ipId: %s, config: %+v", ipid, ipStatus) + } else { + if ipStatus.IPSubnet != secondaryIpConfig.IPSubnet { + t.Fatalf("IPId: %s IPSubnet doesnt match: expected %+v, actual: %+v", ipid, secondaryIpConfig.IPSubnet, ipStatus.IPSubnet) + } + + // Validate IP state + if ipStatus.State != cns.Available { + t.Fatalf("IPId: %s State is not Available, ipStatus: %+v", ipid, ipStatus) + } + + alreadyValidated[ipid] = ipStatus.IPSubnet.IPAddress + } + } else { + // if ipaddress is not same, then fail + if ipaddress != ipStatus.IPSubnet.IPAddress { + t.Fatalf("Added the same IP guid :%s with different ipaddress, expected:%s, actual %s", ipid, ipStatus.IPSubnet.IPAddress, ipaddress) + } + } + } +} + +func generateNetworkContainerRequest(secondaryIps map[string]cns.SecondaryIPConfig) cns.CreateNetworkContainerRequest{ + var ipConfig cns.IPConfiguration + ipConfig.DNSServers = []string{"8.8.8.8", "8.8.4.4"} + ipConfig.GatewayIPAddress = gatewayIp + var ipSubnet cns.IPSubnet + ipSubnet.IPAddress = primaryIp + ipSubnet.PrefixLength = 32 + ipConfig.IPSubnet = ipSubnet + + req := cns.CreateNetworkContainerRequest{ + NetworkContainerType: dockerContainerType, + NetworkContainerid: "testNcId1", + IPConfiguration: ipConfig, + } + + req.SecondaryIPConfigs = make(map[string]cns.SecondaryIPConfig) + for k, v := range secondaryIps { + req.SecondaryIPConfigs[k] = v + } + + return req +} \ No newline at end of file diff --git a/cns/restserver/ipam.go b/cns/restserver/ipam.go index ae933d506c..44b158c1d8 100644 --- a/cns/restserver/ipam.go +++ b/cns/restserver/ipam.go @@ -12,42 +12,12 @@ import ( "github.com/Azure/azure-container-networking/cns/logger" ) -func newIPConfig(ipAddress string, prefixLength uint8) cns.IPSubnet { - return cns.IPSubnet{ - IPAddress: ipAddress, - PrefixLength: prefixLength, - } -} - -func NewPodState(ipaddress string, prefixLength uint8, id, ncid, state string) cns.ContainerIPConfigState { - ipconfig := newIPConfig(ipaddress, prefixLength) - - return cns.ContainerIPConfigState{ - IPConfig: ipconfig, - ID: id, - NCID: ncid, - State: state, - } -} - -func NewPodStateWithOrchestratorContext(ipaddress string, prefixLength uint8, id, ncid, state string, orchestratorContext cns.KubernetesPodInfo) (cns.ContainerIPConfigState, error) { - ipconfig := newIPConfig(ipaddress, prefixLength) - b, err := json.Marshal(orchestratorContext) - return cns.ContainerIPConfigState{ - IPConfig: ipconfig, - ID: id, - NCID: ncid, - State: state, - OrchestratorContext: b, - }, err -} - // used to request an IPConfig from the CNS state func (service *HTTPRestService) requestIPConfigHandler(w http.ResponseWriter, r *http.Request) { var ( err error ipconfigRequest cns.GetIPConfigRequest - ipState cns.ContainerIPConfigState + ipState ipConfigurationStatus returnCode int returnMessage string ) @@ -72,7 +42,7 @@ func (service *HTTPRestService) requestIPConfigHandler(w http.ResponseWriter, r reserveResp := &cns.GetIPConfigResponse{ Response: resp, } - reserveResp.IPConfiguration.IPSubnet = ipState.IPConfig + reserveResp.IPConfiguration.IPSubnet = ipState.IPSubnet err = service.Listener.Encode(w, &reserveResp) logger.Response(service.Name, reserveResp, resp.ReturnCode, ReturnCodeToString(resp.ReturnCode), err) @@ -105,7 +75,7 @@ func (service *HTTPRestService) releaseIPConfigHandler(w http.ResponseWriter, r logger.Response(service.Name, resp, resp.ReturnCode, ReturnCodeToString(resp.ReturnCode), err) }() - if service.state.OrchestratorType != cns.Kubernetes { + if service.state.OrchestratorType != cns.KubernetesCRD { err = fmt.Errorf("ReleaseIPConfig API supported only for kubernetes orchestrator") return } @@ -122,132 +92,34 @@ func (service *HTTPRestService) releaseIPConfigHandler(w http.ResponseWriter, r return } -func validateIPConfig(ipconfig cns.ContainerIPConfigState) error { - if ipconfig.ID == "" { - return fmt.Errorf("Failed to add IPConfig to state: %+v, empty ID", ipconfig) - } - if ipconfig.State == "" { - return fmt.Errorf("Failed to add IPConfig to state: %+v, empty State", ipconfig) - } - if ipconfig.IPConfig.IPAddress == "" { - return fmt.Errorf("Failed to add IPConfig to state: %+v, empty IPSubnet.IPAddress", ipconfig) - } - if ipconfig.IPConfig.PrefixLength == 0 { - return fmt.Errorf("Failed to add IPConfig to state: %+v, empty IPSubnet.PrefixLength", ipconfig) - } - return nil -} - -func (service *HTTPRestService) CreateOrUpdateNetworkContainerWithSecondaryIPConfigs(nc cns.CreateNetworkContainerRequest) error { - //return service.addIPConfigsToState(nc.SecondaryIPConfigs) - return nil -} - -//AddIPConfigsToState takes a lock on the service object, and will add an array of ipconfigs to the CNS Service. -//Used to add IPConfigs to the CNS pool, specifically in the scenario of rebatching. -func (service *HTTPRestService) addIPConfigsToState(ipconfigs map[string]cns.ContainerIPConfigState) error { - var ( - err error - ipconfig cns.ContainerIPConfigState - ) - - addedIPconfigs := make([]cns.ContainerIPConfigState, 0) - - service.Lock() - - defer func() { - service.Unlock() - - if err != nil { - if removeErr := service.removeIPConfigsFromState(addedIPconfigs); removeErr != nil { - logger.Printf("Failed remove IPConfig after AddIpConfigs: %v", removeErr) - } - } - }() - - // ensure the ipconfigs we are not attempting to overwrite existing ipconfig state - existingIPConfigs := filterIPConfigMap(ipconfigs, func(ipconfig *cns.ContainerIPConfigState) bool { - existingIPConfig, exists := service.PodIPConfigState[ipconfig.ID] - if exists && existingIPConfig.State != ipconfig.State { - return true - } - return false - }) - if len(existingIPConfigs) > 0 { - return fmt.Errorf("Failed to add IPConfigs to state, attempting to overwrite existing ipconfig states: %v", existingIPConfigs) - } - - // add ipconfigs to state - for _, ipconfig = range ipconfigs { - if err = validateIPConfig(ipconfig); err != nil { - return err - } - - service.PodIPConfigState[ipconfig.ID] = ipconfig - addedIPconfigs = append(addedIPconfigs, ipconfig) - - if ipconfig.State == cns.Allocated { - var podInfo cns.KubernetesPodInfo - - if err = json.Unmarshal(ipconfig.OrchestratorContext, &podInfo); err != nil { - return fmt.Errorf("Failed to add IPConfig to state: %+v with error: %v", ipconfig, err) - } - - service.PodIPIDByOrchestratorContext[podInfo.GetOrchestratorContextKey()] = ipconfig.ID - } - } - return err -} - -func filterIPConfigMap(toBeAdded map[string]cns.ContainerIPConfigState, f func(*cns.ContainerIPConfigState) bool) []*cns.ContainerIPConfigState { - vsf := make([]*cns.ContainerIPConfigState, 0) - for _, v := range toBeAdded { - if f(&v) { - vsf = append(vsf, &v) - } - } - return vsf -} - -func (service *HTTPRestService) GetAllocatedIPConfigs() []*cns.ContainerIPConfigState { +func (service *HTTPRestService) GetAllocatedIPConfigs() []ipConfigurationStatus { service.RLock() defer service.RUnlock() - return filterIPConfigMap(service.PodIPConfigState, func(ipconfig *cns.ContainerIPConfigState) bool { + return filterIPConfigMap(service.PodIPConfigState, func(ipconfig ipConfigurationStatus) bool { return ipconfig.State == cns.Allocated }) } -func (service *HTTPRestService) GetAvailableIPConfigs() []*cns.ContainerIPConfigState { +func (service *HTTPRestService) GetAvailableIPConfigs() []ipConfigurationStatus { service.RLock() defer service.RUnlock() - return filterIPConfigMap(service.PodIPConfigState, func(ipconfig *cns.ContainerIPConfigState) bool { + return filterIPConfigMap(service.PodIPConfigState, func(ipconfig ipConfigurationStatus) bool { return ipconfig.State == cns.Available }) } -//RemoveIPConfigsFromState takes a lock on the service object, and will remove an array of ipconfigs to the CNS Service. -//Used to add IPConfigs to the CNS pool, specifically in the scenario of rebatching. -func (service *HTTPRestService) removeIPConfigsFromState(ipconfigs []cns.ContainerIPConfigState) error { - service.Lock() - defer service.Unlock() - - for _, ipconfig := range ipconfigs { - delete(service.PodIPConfigState, ipconfig.ID) - var podInfo cns.KubernetesPodInfo - err := json.Unmarshal(ipconfig.OrchestratorContext, &podInfo) - - // if batch delete failed return - if err != nil { - return err +func filterIPConfigMap(toBeAdded map[string]ipConfigurationStatus, f func(ipConfigurationStatus) bool) []ipConfigurationStatus { + vsf := make([]ipConfigurationStatus, 0) + for _, v := range toBeAdded { + if f(v) { + vsf = append(vsf, v) } - - delete(service.PodIPIDByOrchestratorContext, podInfo.GetOrchestratorContextKey()) } - return nil + return vsf } //SetIPConfigAsAllocated takes a lock of the service, and sets the ipconfig in the CNS state as allocated, does not take a lock -func (service *HTTPRestService) setIPConfigAsAllocated(ipconfig cns.ContainerIPConfigState, podInfo cns.KubernetesPodInfo, marshalledOrchestratorContext json.RawMessage) cns.ContainerIPConfigState { +func (service *HTTPRestService) setIPConfigAsAllocated(ipconfig ipConfigurationStatus, podInfo cns.KubernetesPodInfo, marshalledOrchestratorContext json.RawMessage) ipConfigurationStatus { ipconfig.State = cns.Allocated ipconfig.OrchestratorContext = marshalledOrchestratorContext service.PodIPIDByOrchestratorContext[podInfo.GetOrchestratorContextKey()] = ipconfig.ID @@ -256,7 +128,7 @@ func (service *HTTPRestService) setIPConfigAsAllocated(ipconfig cns.ContainerIPC } //SetIPConfigAsAllocated and sets the ipconfig in the CNS state as allocated, does not take a lock -func (service *HTTPRestService) setIPConfigAsAvailable(ipconfig cns.ContainerIPConfigState, podInfo cns.KubernetesPodInfo) cns.ContainerIPConfigState { +func (service *HTTPRestService) setIPConfigAsAvailable(ipconfig ipConfigurationStatus, podInfo cns.KubernetesPodInfo) ipConfigurationStatus { ipconfig.State = cns.Available ipconfig.OrchestratorContext = nil service.PodIPConfigState[ipconfig.ID] = ipconfig @@ -265,6 +137,8 @@ func (service *HTTPRestService) setIPConfigAsAvailable(ipconfig cns.ContainerIPC } ////SetIPConfigAsAllocated takes a lock of the service, and sets the ipconfig in the CNS stateas Available +// Todo - CNI should also pass the IPAddress which needs to be released to validate if that is the right IP allcoated +// in the first place. func (service *HTTPRestService) ReleaseIPConfig(podInfo cns.KubernetesPodInfo) error { service.Lock() defer service.Unlock() @@ -284,9 +158,9 @@ func (service *HTTPRestService) ReleaseIPConfig(podInfo cns.KubernetesPodInfo) e return nil } -func (service *HTTPRestService) GetExistingIPConfig(podInfo cns.KubernetesPodInfo) (cns.ContainerIPConfigState, bool, error) { +func (service *HTTPRestService) GetExistingIPConfig(podInfo cns.KubernetesPodInfo) (ipConfigurationStatus, bool, error) { var ( - ipState cns.ContainerIPConfigState + ipState ipConfigurationStatus isExist bool ) @@ -305,14 +179,14 @@ func (service *HTTPRestService) GetExistingIPConfig(podInfo cns.KubernetesPodInf return ipState, isExist, nil } -func (service *HTTPRestService) AllocateDesiredIPConfig(podInfo cns.KubernetesPodInfo, desiredIPAddress string, orchestratorContext json.RawMessage) (cns.ContainerIPConfigState, error) { - var ipState cns.ContainerIPConfigState +func (service *HTTPRestService) AllocateDesiredIPConfig(podInfo cns.KubernetesPodInfo, desiredIPAddress string, orchestratorContext json.RawMessage) (ipConfigurationStatus, error) { + var ipState ipConfigurationStatus service.Lock() defer service.Unlock() for _, ipState := range service.PodIPConfigState { - if ipState.IPConfig.IPAddress == desiredIPAddress { + if ipState.IPSubnet.IPAddress == desiredIPAddress { if ipState.State == cns.Available { return service.setIPConfigAsAllocated(ipState, podInfo, orchestratorContext), nil } @@ -322,8 +196,8 @@ func (service *HTTPRestService) AllocateDesiredIPConfig(podInfo cns.KubernetesPo return ipState, fmt.Errorf("Requested IP not found in pool") } -func (service *HTTPRestService) AllocateAnyAvailableIPConfig(podInfo cns.KubernetesPodInfo, orchestratorContext json.RawMessage) (cns.ContainerIPConfigState, error) { - var ipState cns.ContainerIPConfigState +func (service *HTTPRestService) AllocateAnyAvailableIPConfig(podInfo cns.KubernetesPodInfo, orchestratorContext json.RawMessage) (ipConfigurationStatus, error) { + var ipState ipConfigurationStatus service.Lock() defer service.Unlock() @@ -337,15 +211,16 @@ func (service *HTTPRestService) AllocateAnyAvailableIPConfig(podInfo cns.Kuberne } // If IPConfig is already allocated for pod, it returns that else it returns one of the available ipconfigs. -func requestIPConfigHelper(service *HTTPRestService, req cns.GetIPConfigRequest) (cns.ContainerIPConfigState, error) { +func requestIPConfigHelper(service *HTTPRestService, req cns.GetIPConfigRequest) (ipConfigurationStatus, error) { var ( podInfo cns.KubernetesPodInfo - ipState cns.ContainerIPConfigState + ipState ipConfigurationStatus isExist bool err error ) - if service.state.OrchestratorType != cns.Kubernetes { + // todo - change it to + if service.state.OrchestratorType != cns.KubernetesCRD { return ipState, fmt.Errorf("AllocateIPconfig API supported only for kubernetes orchestrator") } diff --git a/cns/restserver/ipam_test.go b/cns/restserver/ipam_test.go index 8df73b5286..217eef76c7 100644 --- a/cns/restserver/ipam_test.go +++ b/cns/restserver/ipam_test.go @@ -5,6 +5,7 @@ package restserver import ( "encoding/json" + "fmt" "reflect" "testing" @@ -29,6 +30,7 @@ var ( PodNamespace: "testpod2namespace", } + testIP3 = "10.0.0.3" testPod3GUID = "718e04ac-5a13-4dce-84b3-040accaa9b41" testPod3Info = cns.KubernetesPodInfo{ PodName: "testpod3", @@ -40,20 +42,71 @@ func getTestService() *HTTPRestService { var config common.ServiceConfig httpsvc, _ := NewHTTPRestService(&config) svc := httpsvc.(*HTTPRestService) - svc.state.OrchestratorType = cns.Kubernetes + svc.state.OrchestratorType = cns.KubernetesCRD return svc } +func newSecondaryIPConfig(ipAddress string, prefixLength uint8) cns.SecondaryIPConfig { + return cns.SecondaryIPConfig{ + IPSubnet: cns.IPSubnet{ + IPAddress: ipAddress, + PrefixLength: prefixLength, + }, + } +} + +func NewPodState(ipaddress string, prefixLength uint8, id, ncid, state string) ipConfigurationStatus { + ipconfig := newSecondaryIPConfig(ipaddress, prefixLength) + + return ipConfigurationStatus{ + IPSubnet: ipconfig.IPSubnet, + ID: id, + NCID: ncid, + State: state, + } +} + +func NewPodStateWithOrchestratorContext(ipaddress string, prefixLength uint8, id, ncid, state string, orchestratorContext cns.KubernetesPodInfo) (ipConfigurationStatus, error) { + ipconfig := newSecondaryIPConfig(ipaddress, prefixLength) + b, err := json.Marshal(orchestratorContext) + return ipConfigurationStatus{ + IPSubnet: ipconfig.IPSubnet, + ID: id, + NCID: ncid, + State: state, + OrchestratorContext: b, + }, err +} + +// Test function to populate the IPConfigState +func UpdatePodIpConfigState(svc *HTTPRestService, ipconfigs map[string]ipConfigurationStatus) error { + // add ipconfigs to state + for ipId, ipconfig := range ipconfigs { + + svc.PodIPConfigState[ipId] = ipconfig + if ipconfig.State == cns.Allocated { + var podInfo cns.KubernetesPodInfo + + if err := json.Unmarshal(ipconfig.OrchestratorContext, &podInfo); err != nil { + return fmt.Errorf("Failed to add IPConfig to state: %+v with error: %v", ipconfig, err) + } + + svc.PodIPIDByOrchestratorContext[podInfo.GetOrchestratorContextKey()] = ipId + } + } + return nil +} + // Want first IP func TestIPAMGetAvailableIPConfig(t *testing.T) { svc := getTestService() testState := NewPodState(testIP1, 24, testPod1GUID, testNCID, cns.Available) - ipconfigs := map[string]cns.ContainerIPConfigState{ + ipconfigs := map[string]ipConfigurationStatus{ testState.ID: testState, } - svc.addIPConfigsToState(ipconfigs) + UpdatePodIpConfigState(svc, ipconfigs) req := cns.GetIPConfigRequest{} b, _ := json.Marshal(testPod1Info) @@ -81,11 +134,11 @@ func TestIPAMGetNextAvailableIPConfig(t *testing.T) { state1, _ := NewPodStateWithOrchestratorContext(testIP1, 24, testPod1GUID, testNCID, cns.Allocated, testPod1Info) state2 := NewPodState(testIP2, 24, testPod2GUID, testNCID, cns.Available) - ipconfigs := map[string]cns.ContainerIPConfigState{ + ipconfigs := map[string]ipConfigurationStatus{ state1.ID: state1, state2.ID: state2, } - err := svc.addIPConfigsToState(ipconfigs) + err := UpdatePodIpConfigState(svc, ipconfigs) if err != nil { t.Fatalf("Expected to not fail adding IP's to state: %+v", err) } @@ -111,10 +164,10 @@ func TestIPAMGetAlreadyAllocatedIPConfigForSamePod(t *testing.T) { // Add Allocated Pod IP to state testState, _ := NewPodStateWithOrchestratorContext(testIP1, 24, testPod1GUID, testNCID, cns.Allocated, testPod1Info) - ipconfigs := map[string]cns.ContainerIPConfigState{ + ipconfigs := map[string]ipConfigurationStatus{ testState.ID: testState, } - err := svc.addIPConfigsToState(ipconfigs) + err := UpdatePodIpConfigState(svc, ipconfigs) if err != nil { t.Fatalf("Expected to not fail adding IP's to state: %+v", err) } @@ -140,11 +193,11 @@ func TestIPAMAttemptToRequestIPNotFoundInPool(t *testing.T) { // Add Available Pod IP to state testState := NewPodState(testIP1, 24, testPod1GUID, testNCID, cns.Available) - ipconfigs := map[string]cns.ContainerIPConfigState{ + ipconfigs := map[string]ipConfigurationStatus{ testState.ID: testState, } - err := svc.addIPConfigsToState(ipconfigs) + err := UpdatePodIpConfigState(svc, ipconfigs) if err != nil { t.Fatalf("Expected to not fail adding IP's to state: %+v", err) } @@ -152,7 +205,10 @@ func TestIPAMAttemptToRequestIPNotFoundInPool(t *testing.T) { req := cns.GetIPConfigRequest{} b, _ := json.Marshal(testPod2Info) req.OrchestratorContext = b - req.DesiredIPConfig = newIPConfig(testIP2, 24) + req.DesiredIPConfig = cns.IPSubnet{ + IPAddress: testIP2, + PrefixLength: 24, + } _, err = requestIPConfigHelper(svc, req) if err == nil { @@ -165,11 +221,11 @@ func TestIPAMGetDesiredIPConfigWithSpecfiedIP(t *testing.T) { // Add Available Pod IP to state testState := NewPodState(testIP1, 24, testPod1GUID, testNCID, cns.Available) - ipconfigs := map[string]cns.ContainerIPConfigState{ + ipconfigs := map[string]ipConfigurationStatus{ testState.ID: testState, } - err := svc.addIPConfigsToState(ipconfigs) + err := UpdatePodIpConfigState(svc, ipconfigs) if err != nil { t.Fatalf("Expected to not fail adding IP's to state: %+v", err) } @@ -177,7 +233,10 @@ func TestIPAMGetDesiredIPConfigWithSpecfiedIP(t *testing.T) { req := cns.GetIPConfigRequest{} b, _ := json.Marshal(testPod1Info) req.OrchestratorContext = b - req.DesiredIPConfig = newIPConfig(testIP1, 24) + req.DesiredIPConfig = cns.IPSubnet{ + IPAddress: testIP1, + PrefixLength: 24, + } actualstate, err := requestIPConfigHelper(svc, req) if err != nil { @@ -197,10 +256,10 @@ func TestIPAMFailToGetDesiredIPConfigWithAlreadyAllocatedSpecfiedIP(t *testing.T // set state as already allocated testState, _ := NewPodStateWithOrchestratorContext(testIP1, 24, testPod1GUID, testNCID, cns.Allocated, testPod1Info) - ipconfigs := map[string]cns.ContainerIPConfigState{ + ipconfigs := map[string]ipConfigurationStatus{ testState.ID: testState, } - err := svc.addIPConfigsToState(ipconfigs) + err := UpdatePodIpConfigState(svc, ipconfigs) if err != nil { t.Fatalf("Expected to not fail adding IP's to state: %+v", err) } @@ -209,7 +268,10 @@ func TestIPAMFailToGetDesiredIPConfigWithAlreadyAllocatedSpecfiedIP(t *testing.T req := cns.GetIPConfigRequest{} b, _ := json.Marshal(testPod2Info) req.OrchestratorContext = b - req.DesiredIPConfig = newIPConfig(testIP1, 24) + req.DesiredIPConfig = cns.IPSubnet{ + IPAddress: testIP1, + PrefixLength: 24, + } _, err = requestIPConfigHelper(svc, req) if err == nil { @@ -224,11 +286,11 @@ func TestIPAMFailToGetIPWhenAllIPsAreAllocated(t *testing.T) { state1, _ := NewPodStateWithOrchestratorContext(testIP1, 24, testPod1GUID, testNCID, cns.Allocated, testPod1Info) state2, _ := NewPodStateWithOrchestratorContext(testIP2, 24, testPod2GUID, testNCID, cns.Allocated, testPod2Info) - ipconfigs := map[string]cns.ContainerIPConfigState{ + ipconfigs := map[string]ipConfigurationStatus{ state1.ID: state1, state2.ID: state2, } - err := svc.addIPConfigsToState(ipconfigs) + err := UpdatePodIpConfigState(svc, ipconfigs) if err != nil { t.Fatalf("Expected to not fail adding IP's to state: %+v", err) } @@ -253,16 +315,19 @@ func TestIPAMRequestThenReleaseThenRequestAgain(t *testing.T) { // set state as already allocated state1, _ := NewPodStateWithOrchestratorContext(testIP1, 24, testPod1GUID, testNCID, cns.Allocated, testPod1Info) - ipconfigs := map[string]cns.ContainerIPConfigState{ + ipconfigs := map[string]ipConfigurationStatus{ state1.ID: state1, } - err := svc.addIPConfigsToState(ipconfigs) + err := UpdatePodIpConfigState(svc, ipconfigs) if err != nil { t.Fatalf("Expected to not fail adding IP's to state: %+v", err) } - desiredIPConfig := newIPConfig(testIP1, 24) + desiredIPConfig := cns.IPSubnet{ + IPAddress: testIP1, + PrefixLength: 24, + } // Use TestPodInfo2 to request TestIP1, which has already been allocated req := cns.GetIPConfigRequest{} @@ -294,7 +359,7 @@ func TestIPAMRequestThenReleaseThenRequestAgain(t *testing.T) { desiredState, _ := NewPodStateWithOrchestratorContext(testIP1, 24, testPod1GUID, testNCID, cns.Allocated, testPod1Info) // want first available Pod IP State - desiredState.IPConfig = desiredIPConfig + desiredState.IPSubnet = desiredIPConfig desiredState.OrchestratorContext = b if reflect.DeepEqual(desiredState, actualstate) != true { @@ -302,106 +367,15 @@ func TestIPAMRequestThenReleaseThenRequestAgain(t *testing.T) { } } -func TestIPAMExpectFailWhenAddingBadIPConfig(t *testing.T) { - svc := getTestService() - - var err error - - // set state as already allocated - state1, _ := NewPodStateWithOrchestratorContext(testIP1, 24, testPod1GUID, testNCID, cns.Available, testPod1Info) - - ipconfigs := map[string]cns.ContainerIPConfigState{ - state1.ID: state1, - } - - err = svc.addIPConfigsToState(ipconfigs) - if err != nil { - t.Fatalf("Expected to not fail when good ipconfig is added") - } - - // create bad ipconfig - state2, _ := NewPodStateWithOrchestratorContext("", 24, "", testNCID, cns.Available, testPod1Info) - - ipconfigs2 := map[string]cns.ContainerIPConfigState{ - state2.ID: state2, - } - - // add a bad ipconfig - err = svc.addIPConfigsToState(ipconfigs2) - if err == nil { - t.Fatalf("Expected add to fail when bad ipconfig is added.") - } - - // ensure state remains untouched - if len(svc.PodIPConfigState) != 1 { - t.Fatalf("Expected bad ipconfig to not be added added.") - } -} - -func TestIPAMStateCleanUpWhenAddingGoodIPConfigWithBadOrchestratorContext(t *testing.T) { - svc := getTestService() - - var err error - - // add available state - state1, _ := NewPodStateWithOrchestratorContext(testIP1, 24, testPod1GUID, testNCID, cns.Available, testPod1Info) - - ipconfigs := map[string]cns.ContainerIPConfigState{ - state1.ID: state1, - } - - err = svc.addIPConfigsToState(ipconfigs) - if err != nil { - t.Fatalf("Expected to not fail when good ipconfig is added") - } - - // create a good ipconfig - state2, _ := NewPodStateWithOrchestratorContext(testIP2, 24, testPod2GUID, testNCID, cns.Allocated, testPod1Info) - - // make it bad with a bad orchestratorcontext and add to good ipconfig - b, err := json.Marshal("badstring") - state2.OrchestratorContext = b - - ipconfigs2 := map[string]cns.ContainerIPConfigState{ - state2.ID: state2, - } - - err = svc.addIPConfigsToState(ipconfigs2) - if err == nil { - t.Fatalf("Expected add to fail when bad ipconfig is added.") - } - - // ensure state remains untouched - if len(svc.PodIPConfigState) != 1 { - t.Fatalf("Expected bad ipconfig to not be added added.") - } - - // ensure we can still get the available ipconfig - req := cns.GetIPConfigRequest{} - b, _ = json.Marshal(testPod1Info) - req.OrchestratorContext = b - actualstate, err := requestIPConfigHelper(svc, req) - if err != nil { - t.Fatalf("Expected IP retrieval to be nil: %v", err) - } - - desiredState, _ := NewPodStateWithOrchestratorContext(testIP1, 24, testPod1GUID, testNCID, cns.Allocated, testPod1Info) - desiredState.OrchestratorContext = b - - if reflect.DeepEqual(desiredState, actualstate) != true { - t.Fatalf("Desired state not matching actual state, expected: %+v, actual: %+v", desiredState, actualstate) - } -} - func TestIPAMReleaseIPIdempotency(t *testing.T) { svc := getTestService() // set state as already allocated state1, _ := NewPodStateWithOrchestratorContext(testIP1, 24, testPod1GUID, testNCID, cns.Allocated, testPod1Info) - ipconfigs := map[string]cns.ContainerIPConfigState{ + ipconfigs := map[string]ipConfigurationStatus{ state1.ID: state1, } - err := svc.addIPConfigsToState(ipconfigs) + err := UpdatePodIpConfigState(svc, ipconfigs) if err != nil { t.Fatalf("Expected to not fail adding IP's to state: %+v", err) } @@ -423,61 +397,87 @@ func TestIPAMAllocateIPIdempotency(t *testing.T) { svc := getTestService() // set state as already allocated state1, _ := NewPodStateWithOrchestratorContext(testIP1, 24, testPod1GUID, testNCID, cns.Allocated, testPod1Info) - ipconfigs := map[string]cns.ContainerIPConfigState{ + + ipconfigs := map[string]ipConfigurationStatus{ state1.ID: state1, } - err := svc.addIPConfigsToState(ipconfigs) + err := UpdatePodIpConfigState(svc, ipconfigs) if err != nil { t.Fatalf("Expected to not fail adding IP's to state: %+v", err) } - err = svc.addIPConfigsToState(ipconfigs) + err = UpdatePodIpConfigState(svc, ipconfigs) if err != nil { t.Fatalf("Expected to not fail adding IP's to state: %+v", err) } } -func TestIPAMExpectStateToNotChangeWhenChangingAllocatedToAvailable(t *testing.T) { +func TestAvailableIPConfigs(t *testing.T) { svc := getTestService() - // add two ipconfigs, one as available, the other as allocated - state1, _ := NewPodStateWithOrchestratorContext(testIP1, 24, testPod1GUID, testNCID, cns.Available, testPod1Info) - state2, _ := NewPodStateWithOrchestratorContext(testIP2, 24, testPod2GUID, testNCID, cns.Allocated, testPod2Info) - ipconfigs := map[string]cns.ContainerIPConfigState{ + state1 := NewPodState(testIP1, 24, testPod1GUID, testNCID, cns.Available) + state2 := NewPodState(testIP2, 24, testPod2GUID, testNCID, cns.Available) + state3 := NewPodState(testIP3, 24, testPod3GUID, testNCID, cns.Available) + + ipconfigs := map[string]ipConfigurationStatus{ state1.ID: state1, state2.ID: state2, + state3.ID: state3, } + UpdatePodIpConfigState(svc, ipconfigs) - err := svc.addIPConfigsToState(ipconfigs) - if err != nil { - t.Fatalf("Expected to not fail adding IP's to state: %+v", err) + desiredAvailableIps := map[string]ipConfigurationStatus{ + state1.ID: state1, + state2.ID: state2, + state3.ID: state3, } + availableIps := svc.GetAvailableIPConfigs() + validateIpState(t, availableIps, desiredAvailableIps) - // create state2 again, but as available - state2Available, _ := NewPodStateWithOrchestratorContext(testIP2, 24, testPod2GUID, testNCID, cns.Available, testPod2Info) + desiredAllocatedIpConfigs := make(map[string]ipConfigurationStatus) + allocatedIps := svc.GetAllocatedIPConfigs() + validateIpState(t, allocatedIps, desiredAllocatedIpConfigs) - // add an available and allocated ipconfig - ipconfigsTest := map[string]cns.ContainerIPConfigState{ - state1.ID: state1, - state2.ID: state2Available, - } + req := cns.GetIPConfigRequest{} + b, _ := json.Marshal(testPod1Info) + req.OrchestratorContext = b + req.DesiredIPConfig = state1.IPSubnet - // expect to fail overwriting an allocated state with available - err = svc.addIPConfigsToState(ipconfigsTest) - if err == nil { - t.Fatalf("Expected to fail when overwriting an allocated state as available: %+v", err) + _, err := requestIPConfigHelper(svc, req) + if err != nil { + t.Fatal("Expected IP retrieval to be nil") } - // get allocated ipconfigs, should only be one from the inital call, and not 2 from the failed call - availableIPconfigs := svc.GetAvailableIPConfigs() - if len(availableIPconfigs) != 1 { - t.Fatalf("More than expected available IP configs in state") + delete(desiredAvailableIps, state1.ID) + availableIps = svc.GetAvailableIPConfigs() + validateIpState(t, availableIps, desiredAvailableIps) + + desiredState := NewPodState(testIP1, 24, testPod1GUID, testNCID, cns.Allocated) + desiredState.OrchestratorContext = b + desiredAllocatedIpConfigs[desiredState.ID] = desiredState + allocatedIps = svc.GetAllocatedIPConfigs() + validateIpState(t, allocatedIps, desiredAllocatedIpConfigs) + +} + +func validateIpState(t *testing.T, actualIps []ipConfigurationStatus, expectedList map[string]ipConfigurationStatus) { + if len(actualIps) != len(expectedList) { + t.Fatalf("Actual and expected count doesnt match, expected %d, actual %d", len(actualIps), len(expectedList)) } - // get allocated ipconfigs, should only be one from the inital call, and not 0 from the failed call - allocatedIPconfigs := svc.GetAllocatedIPConfigs() - if len(allocatedIPconfigs) != 1 { - t.Fatalf("More than expected allocated IP configs in state") + for _, actualIp := range actualIps { + var expectedIp ipConfigurationStatus + var found bool + for _, expectedIp = range expectedList { + if reflect.DeepEqual(actualIp, expectedIp) == true { + found = true + break + } + } + + if !found { + t.Fatalf("Actual and expected list doesnt match actual: %+v, expected: %+v", actualIp, expectedIp) + } } } diff --git a/cns/restserver/restserver.go b/cns/restserver/restserver.go index 8b25e9a9dc..0807cb4b1e 100644 --- a/cns/restserver/restserver.go +++ b/cns/restserver/restserver.go @@ -4,6 +4,7 @@ package restserver import ( + "encoding/json" "sync" "time" @@ -23,6 +24,7 @@ import ( // all HTTP APIs - api.go and/or ipam.go // APIs for internal consumption - internalapi.go // All helper/utility functions - util.go +// Constants - const.go var ( // Named Lock for accessing different states in httpRestServiceState @@ -36,9 +38,9 @@ type HTTPRestService struct { imdsClient *imdsclient.ImdsClient ipamClient *ipamclient.IpamClient networkContainer *networkcontainers.NetworkContainers - PodIPIDByOrchestratorContext map[string]string // OrchestratorContext is key and value is Pod IP uuid. - PodIPConfigState map[string]cns.ContainerIPConfigState // seondaryipid(uuid) is key - AllocatedIPCount map[string]allocatedIPCount // key - ncid + PodIPIDByOrchestratorContext map[string]string // OrchestratorContext is key and value is Pod IP uuid. + PodIPConfigState map[string]ipConfigurationStatus // seondaryipid(uuid) is key + AllocatedIPCount map[string]allocatedIPCount // key - ncid routingTable *routes.RoutingTable store store.KeyValueStore state *httpRestServiceState @@ -50,6 +52,16 @@ type allocatedIPCount struct { Count int } +// This is used for KubernetesCRD orchastrator Type where NC has multiple ips. +// This struct captures the state for SecondaryIPs associated to a given NC +type ipConfigurationStatus struct { + NCID string + ID string //uuid + IPSubnet cns.IPSubnet + State string + OrchestratorContext json.RawMessage +} + // containerstatus is used to save status of an existing container type containerstatus struct { ID string @@ -110,7 +122,7 @@ func NewHTTPRestService(config *common.ServiceConfig) (HTTPService, error) { serviceState.joinedNetworks = make(map[string]struct{}) podIPIDByOrchestratorContext := make(map[string]string) - podIPConfigState := make(map[string]cns.ContainerIPConfigState) + podIPConfigState := make(map[string]ipConfigurationStatus) allocatedIPCount := make(map[string]allocatedIPCount) // key - ncid return &HTTPRestService{ diff --git a/cns/restserver/util.go b/cns/restserver/util.go index af39cdaa28..16beba3c58 100644 --- a/cns/restserver/util.go +++ b/cns/restserver/util.go @@ -103,23 +103,18 @@ func (service *HTTPRestService) saveNetworkContainerGoalState(req cns.CreateNetw service.Lock() defer service.Unlock() - existing, ok := service.state.ContainerStatus[req.NetworkContainerid] + existingNCStatus, ok := service.state.ContainerStatus[req.NetworkContainerid] var hostVersion string + var existingSecondaryIPConfigs map[string]cns.SecondaryIPConfig //uuid is key if ok { - hostVersion = existing.HostVersion + hostVersion = existingNCStatus.HostVersion + existingSecondaryIPConfigs = existingNCStatus.CreateNetworkContainerRequest.SecondaryIPConfigs } if service.state.ContainerStatus == nil { service.state.ContainerStatus = make(map[string]containerstatus) } - service.state.ContainerStatus[req.NetworkContainerid] = - containerstatus{ - ID: req.NetworkContainerid, - VMVersion: req.Version, - CreateNetworkContainerRequest: req, - HostVersion: hostVersion} - switch req.NetworkContainerType { case cns.AzureContainerInstance: fallthrough @@ -143,7 +138,7 @@ func (service *HTTPRestService) saveNetworkContainerGoalState(req cns.CreateNetw fallthrough case cns.AzureFirstParty: fallthrough - case cns.WebApps: + case cns.WebApps: // todo: Is WebApps an OrchastratorType or ContainerType? var podInfo cns.KubernetesPodInfo err := json.Unmarshal(req.OrchestratorContext, &podInfo) if err != nil { @@ -160,21 +155,138 @@ func (service *HTTPRestService) saveNetworkContainerGoalState(req cns.CreateNetw service.state.ContainerIDByOrchestratorContext[podInfo.PodName+podInfo.PodNamespace] = req.NetworkContainerid break + case cns.KubernetesCRD: + // Validate and Update the SecondaryIpConfig state + returnCode, returnMesage := service.updateIpConfigsStateUntransacted(req, existingSecondaryIPConfigs) + if returnCode != 0 { + return returnCode, returnMesage + } default: errMsg := fmt.Sprintf("Unsupported orchestrator type: %s", service.state.OrchestratorType) logger.Errorf(errMsg) return UnsupportedOrchestratorType, errMsg } + default: errMsg := fmt.Sprintf("Unsupported network container type %s", req.NetworkContainerType) logger.Errorf(errMsg) return UnsupportedNetworkContainerType, errMsg } + service.state.ContainerStatus[req.NetworkContainerid] = + containerstatus{ + ID: req.NetworkContainerid, + VMVersion: req.Version, + CreateNetworkContainerRequest: req, + HostVersion: hostVersion} + service.saveState() return 0, "" } +// This func will compute the deltaIpConfigState which needs to be updated (Added or Deleted) +// from the inmemory map +// Note: Also this func is an untransacted API as the caller will take a Service lock +func (service *HTTPRestService) updateIpConfigsStateUntransacted(req cns.CreateNetworkContainerRequest, existingSecondaryIPConfigs map[string]cns.SecondaryIPConfig) (int, string) { + // parse the existingSecondaryIpConfigState to find the deleted Ips + newIPConfigs := req.SecondaryIPConfigs + var tobeDeletedIpConfigs = make(map[string]cns.SecondaryIPConfig) + + // Populate the ToBeDeleted list, Secondary IPs which doesnt exist in New request anymore. + // We will later remove them from the in-memory cache + for secondaryIpId, existingIPConfig := range existingSecondaryIPConfigs { + _, exists := newIPConfigs[secondaryIpId] + if !exists { + // IP got removed in the updated request, add it in tobeDeletedIps + tobeDeletedIpConfigs[secondaryIpId] = existingIPConfig + } + } + + // Validate TobeDeletedIps are ready to be deleted. + for ipId, _ := range tobeDeletedIpConfigs { + ipConfigStatus, exists := service.PodIPConfigState[ipId] + if exists { + // pod ip exists, validate if state is not allocated, else fail + if ipConfigStatus.State == cns.Allocated { + errMsg := fmt.Sprintf("Failed to delete an Allocated IP %v", ipConfigStatus) + return InconsistentIPConfigState, errMsg + } + } + } + + // now actually remove the deletedIPs + for ipId, _ := range tobeDeletedIpConfigs { + returncode, errMsg := service.removeToBeDeletedIpsStateUntransacted(ipId, true) + if returncode != Success { + return returncode, errMsg + } + } + + // Add the newIpConfigs, ignore if ip state is already in the map + service.addIPConfigStateUntransacted(req.NetworkContainerid, newIPConfigs) + + return 0, "" +} + +// addIPConfigStateUntransacted adds the IPConfis to the PodIpConfigState map with Available state +// If the IP is already added then it will be an idempotent call. Also note, caller will +// acquire/release the service lock. +func (service *HTTPRestService) addIPConfigStateUntransacted(ncId string, ipconfigs map[string]cns.SecondaryIPConfig) { + // add ipconfigs to state + for ipId, ipconfig := range ipconfigs { + // if this IPConfig already exists in the map, then ignore as this is an idempotent state + if _, exists := service.PodIPConfigState[ipId]; exists { + continue + } + + // add the new State + ipconfigStatus := ipConfigurationStatus{ + NCID: ncId, + ID: ipId, + IPSubnet: ipconfig.IPSubnet, + State: cns.Available, + OrchestratorContext: nil, + } + + service.PodIPConfigState[ipId] = ipconfigStatus + + // Todo Update batch API and maintain the count + } +} + +// Todo: call this when request is received +func validateIPSubnet(ipSubnet cns.IPSubnet) error { + if ipSubnet.IPAddress == "" { + return fmt.Errorf("Failed to add IPConfig to state: %+v, empty IPSubnet.IPAddress", ipSubnet) + } + if ipSubnet.PrefixLength == 0 { + return fmt.Errorf("Failed to add IPConfig to state: %+v, empty IPSubnet.PrefixLength", ipSubnet) + } + return nil +} + +// removeToBeDeletedIpsStateUntransacted removes IPConfigs from the PodIpConfigState map +// Caller will acquire/release the service lock. +func (service *HTTPRestService) removeToBeDeletedIpsStateUntransacted(ipId string, skipValidation bool) (int, string) { + + // this is set if caller has already done the validation + if !skipValidation { + ipConfigStatus, exists := service.PodIPConfigState[ipId] + if exists { + // pod ip exists, validate if state is not allocated, else fail + if ipConfigStatus.State == cns.Allocated { + errMsg := fmt.Sprintf("Failed to delete an Allocated IP %v", ipConfigStatus) + return InconsistentIPConfigState, errMsg + } + } + } + + // Delete this ip from PODIpConfigState Map + logger.Printf("[Azure-Cns] Delete the PodIpConfigState, IpId: %s, IPConfigStatus: %v", ipId, service.PodIPConfigState[ipId]) + delete(service.PodIPConfigState, ipId) + return 0, "" +} + func (service *HTTPRestService) getNetworkContainerResponse(req cns.GetNetworkContainerRequest) cns.GetNetworkContainerResponse { var containerID string var getNetworkContainerResponse cns.GetNetworkContainerResponse @@ -411,6 +523,8 @@ func logNCSnapshot(createNetworkContainerRequest cns.CreateNetworkContainerReque aiEvent.Properties[logger.NetworkContainerTypeStr] = createNetworkContainerRequest.NetworkContainerType aiEvent.Properties[logger.OrchestratorContextStr] = fmt.Sprintf("%s", createNetworkContainerRequest.OrchestratorContext) + // TODO - Add for SecondaryIPs (Task: https://msazure.visualstudio.com/One/_workitems/edit/7711831) + logger.LogEvent(aiEvent) }