From dbb4e129b835d2e9c2849ce4d622d5199c5a79e1 Mon Sep 17 00:00:00 2001 From: Kshitija Murudi Date: Wed, 17 Mar 2021 10:01:59 -0700 Subject: [PATCH 1/2] CNS Debug API to expose PodIpIdByOrchrestratorContext This API exposes the Httpservice struct field to display map of OrchestratorContext->PodIpID (UUID) and includes debug command getPodContexts for CLI use-case. --- cns/NetworkContainerContract.go | 7 +++++ cns/cnsclient/cli.go | 23 ++++++++++++++++ cns/cnsclient/cnsclient.go | 47 +++++++++++++++++++++++++++++++++ cns/cnsclient/cnsclient_test.go | 29 +++++++++++++------- cns/restserver/ipam.go | 38 ++++++++++++++++++++++++++ cns/restserver/restserver.go | 1 + 6 files changed, 135 insertions(+), 10 deletions(-) diff --git a/cns/NetworkContainerContract.go b/cns/NetworkContainerContract.go index 5c6dc44c9d..814f2fd221 100644 --- a/cns/NetworkContainerContract.go +++ b/cns/NetworkContainerContract.go @@ -23,6 +23,7 @@ const ( RequestIPConfig = "/network/requestipconfig" ReleaseIPConfig = "/network/releaseipconfig" GetIPAddresses = "/debug/getipaddresses" + GetPodIPOrchestratorContext = "/debug/getpodcontext" ) // NetworkContainer Prefixes @@ -248,6 +249,12 @@ type GetIPAddressStatusResponse struct { Response Response } +//GetPodContextResponse is used in CNS Client debug mode to get mapping of Orchestrator Context to Pod IP UUID +type GetPodContextResponse struct { + PodContext map[string]string + Response Response +} + // IPAddressState Only used in the GetIPConfig API to return IP's that match a filter type IPAddressState struct { IPAddress string diff --git a/cns/cnsclient/cli.go b/cns/cnsclient/cli.go index 72bec087c5..955a6b19b3 100644 --- a/cns/cnsclient/cli.go +++ b/cns/cnsclient/cli.go @@ -15,6 +15,7 @@ const ( getAllocatedArg = "Allocated" getAllArg = "All" getPendingReleaseArg = "PendingRelease" + getPodCmdArg = "getPodContexts" releaseArg = "release" @@ -28,6 +29,7 @@ const ( var ( availableCmds = []string{ getCmdArg, + getPodCmdArg, } getFlags = []string{ @@ -49,6 +51,8 @@ func HandleCNSClientCommands(cmd, arg string) error { switch { case strings.EqualFold(getCmdArg, cmd): return getCmd(cnsClient, arg) + case strings.EqualFold(getPodCmdArg, cmd): + return getPodCmd(cnsClient) default: return fmt.Errorf("No debug cmd supplied, options are: %v", getCmdArg) } @@ -96,3 +100,22 @@ func printIPAddresses(addrSlice []cns.IPConfigurationStatus) { cns.IPConfigurationStatus.String(addr) } } + +func getPodCmd(client *CNSClient) error { + + resp, err := client.GetPodOrchestratorContext() + if err != nil { + return err + } + + printPodContext(resp) + return nil +} + +func printPodContext(podContext map[string]string){ + var i = 1 + for orchContext,podID := range podContext { + fmt.Println(i, " ", orchContext, " : ", podID) + i++ + } +} diff --git a/cns/cnsclient/cnsclient.go b/cns/cnsclient/cnsclient.go index 8d75cbac4a..5e2b8f7f7b 100644 --- a/cns/cnsclient/cnsclient.go +++ b/cns/cnsclient/cnsclient.go @@ -371,3 +371,50 @@ func (cnsClient *CNSClient) GetIPAddressesMatchingStates(StateFilter ...string) return resp.IPConfigurationStatus, err } +//GetPodOrchestratorContext calls GetPodIpOrchestratorContext API on CNS +func (cnsClient *CNSClient) GetPodOrchestratorContext() (map[string]string, error) { + var ( + resp cns.GetPodContextResponse + err error + res *http.Response + body bytes.Buffer + ) + + url := cnsClient.connectionURL + cns.GetPodIPOrchestratorContext + log.Printf("GetPodIPOrchestratorContext url %v", url) + + payload := "" + + err = json.NewEncoder(&body).Encode(payload) + if err != nil { + log.Errorf("encoding json failed with %v", err) + return resp.PodContext, err + } + + res, err = http.Post(url, contentTypeJSON, &body) + if err != nil { + log.Errorf("[Azure CNSClient] HTTP Post returned error %v", err.Error()) + return resp.PodContext, err + } + + defer res.Body.Close() + + if res.StatusCode != http.StatusOK { + errMsg := fmt.Sprintf("[Azure CNSClient] GetPodContext invalid http status code: %v", res.StatusCode) + log.Errorf(errMsg) + return resp.PodContext, fmt.Errorf(errMsg) + } + + err = json.NewDecoder(res.Body).Decode(&resp) + if err != nil { + log.Errorf("[Azure CNSClient] Error received while parsing GetPodContext response resp:%v err:%v", res.Body, err.Error()) + return resp.PodContext, err + } + + if resp.Response.ReturnCode != 0 { + log.Errorf("[Azure CNSClient] GetPodContext received error response :%v", resp.Response.Message) + return resp.PodContext, fmt.Errorf(resp.Response.Message) + } + + return resp.PodContext, err +} diff --git a/cns/cnsclient/cnsclient_test.go b/cns/cnsclient/cnsclient_test.go index dd09dcb03a..d030926f46 100644 --- a/cns/cnsclient/cnsclient_test.go +++ b/cns/cnsclient/cnsclient_test.go @@ -11,7 +11,6 @@ import ( "reflect" "strconv" "testing" - "github.com/Azure/azure-container-networking/cns" "github.com/Azure/azure-container-networking/cns/common" "github.com/Azure/azure-container-networking/cns/fakes" @@ -226,14 +225,8 @@ func TestCNSClientRequestAndRelease(t *testing.T) { if reflect.DeepEqual(desired, resultIPnet) != true { t.Fatalf("Desired result not matching actual result, expected: %+v, actual: %+v", desired, resultIPnet) } - - // release requested IP address, expect success - err = cnsClient.ReleaseIPAddress(orchestratorContext) - if err != nil { - t.Fatalf("Expected to not fail when releasing IP reservation found with context: %+v", err) - } - - ipaddresses, err := cnsClient.GetIPAddressesMatchingStates(cns.Available) + //checking for allocated IP address and pod context printing before ReleaseIPAddress is called + ipaddresses, err := cnsClient.GetIPAddressesMatchingStates(cns.Allocated) if err != nil { t.Fatalf("Get allocated IP addresses failed %+v", err) } @@ -242,8 +235,24 @@ func TestCNSClientRequestAndRelease(t *testing.T) { t.Fatalf("Number of available IP addresses expected to be 1, actual %+v", ipaddresses) } - if ipaddresses[0].IPAddress != desiredIpAddress && ipaddresses[0].State != cns.Available { + if ipaddresses[0].IPAddress != desiredIpAddress && ipaddresses[0].State != cns.Allocated { t.Fatalf("Available IP address does not match expected, address state: %+v", ipaddresses) } fmt.Println(ipaddresses) + + //test for pod ip by orch context map + podcontext, err := cnsClient.GetPodOrchestratorContext() + if err != nil { + t.Fatalf("Get pod ip by orchestrator context failed %+v", err) + } + if len(podcontext) < 1 { + t.Fatalf("Error in getting podcontext %+v", podcontext) + } + fmt.Println(podcontext) + + // release requested IP address, expect success + err = cnsClient.ReleaseIPAddress(orchestratorContext) + if err != nil { + t.Fatalf("Expected to not fail when releasing IP reservation found with context: %+v", err) + } } diff --git a/cns/restserver/ipam.go b/cns/restserver/ipam.go index 156c072b4d..3e61b544c7 100644 --- a/cns/restserver/ipam.go +++ b/cns/restserver/ipam.go @@ -181,6 +181,44 @@ func (service *HTTPRestService) GetPodIPConfigState() map[string]cns.IPConfigura return service.PodIPConfigState } +func (service *HTTPRestService) getPodIPIDByOrchestratorContexthandler(w http.ResponseWriter, r *http.Request) { + var ( + req string + resp cns.GetPodContextResponse + statusCode int + returnMessage string + err error + ) + + statusCode = UnexpectedError + + defer func() { + if err != nil { + resp.Response.ReturnCode = statusCode + resp.Response.Message = returnMessage + } + + err = service.Listener.Encode(w, &resp) + logger.Response(service.Name, resp, resp.Response.ReturnCode, ReturnCodeToString(resp.Response.ReturnCode), err) + }() + + err = service.Listener.Decode(w, r, &req) + if err != nil { + returnMessage = err.Error() + return + } + + resp.PodContext = service.GetPodIPIDByOrchestratorContext() + + return +} + +func (service *HTTPRestService) GetPodIPIDByOrchestratorContext() map[string]string { + service.RLock() + defer service.RUnlock() + return service.PodIPIDByOrchestratorContext +} + // GetPendingProgramIPConfigs returns list of IPs which are in pending program status func (service *HTTPRestService) GetPendingProgramIPConfigs() []cns.IPConfigurationStatus { service.RLock() diff --git a/cns/restserver/restserver.go b/cns/restserver/restserver.go index a3f1b77e0d..ab70e82e95 100644 --- a/cns/restserver/restserver.go +++ b/cns/restserver/restserver.go @@ -174,6 +174,7 @@ func (service *HTTPRestService) Init(config *common.ServiceConfig) error { listener.AddHandler(cns.ReleaseIPConfig, service.releaseIPConfigHandler) listener.AddHandler(cns.NmAgentSupportedApisPath, service.nmAgentSupportedApisHandler) listener.AddHandler(cns.GetIPAddresses, service.getIPAddressesHandler) + listener.AddHandler(cns.GetPodIPOrchestratorContext, service.getPodIPIDByOrchestratorContexthandler) // handlers for v0.2 listener.AddHandler(cns.V2Prefix+cns.SetEnvironmentPath, service.setEnvironment) From 2d318a8d4a6c3cdf8d3972bc527c3b4637844ff9 Mon Sep 17 00:00:00 2001 From: Kshitija Murudi Date: Thu, 25 Mar 2021 12:01:06 -0700 Subject: [PATCH 2/2] CNS Debug API for getting PodContexts Added GET endpoint instead for API Handler instead of POST. Added separate test function. Formatting on multiple files under /cns. --- cns/NetworkContainerContract.go | 6 ++-- cns/cnsclient/cli.go | 8 ++--- cns/cnsclient/cnsclient.go | 16 +++------- cns/cnsclient/cnsclient_test.go | 45 ++++++++++++++++++++++------ cns/restserver/ipam.go | 53 ++++++++++++++------------------- 5 files changed, 70 insertions(+), 58 deletions(-) diff --git a/cns/NetworkContainerContract.go b/cns/NetworkContainerContract.go index 814f2fd221..847249b731 100644 --- a/cns/NetworkContainerContract.go +++ b/cns/NetworkContainerContract.go @@ -245,14 +245,14 @@ type GetIPAddressStateResponse struct { // GetIPAddressStatusResponse is used in CNS IPAM mode as a response to get IP address, state and Pod info type GetIPAddressStatusResponse struct { - IPConfigurationStatus[] IPConfigurationStatus - Response Response + IPConfigurationStatus []IPConfigurationStatus + Response Response } //GetPodContextResponse is used in CNS Client debug mode to get mapping of Orchestrator Context to Pod IP UUID type GetPodContextResponse struct { PodContext map[string]string - Response Response + Response Response } // IPAddressState Only used in the GetIPConfig API to return IP's that match a filter diff --git a/cns/cnsclient/cli.go b/cns/cnsclient/cli.go index 955a6b19b3..a704d7fbef 100644 --- a/cns/cnsclient/cli.go +++ b/cns/cnsclient/cli.go @@ -112,10 +112,10 @@ func getPodCmd(client *CNSClient) error { return nil } -func printPodContext(podContext map[string]string){ +func printPodContext(podContext map[string]string) { var i = 1 - for orchContext,podID := range podContext { - fmt.Println(i, " ", orchContext, " : ", podID) - i++ + for orchContext, podID := range podContext { + fmt.Println(i, " ", orchContext, " : ", podID) + i++ } } diff --git a/cns/cnsclient/cnsclient.go b/cns/cnsclient/cnsclient.go index 5e2b8f7f7b..8e55bcf6cb 100644 --- a/cns/cnsclient/cnsclient.go +++ b/cns/cnsclient/cnsclient.go @@ -371,36 +371,28 @@ func (cnsClient *CNSClient) GetIPAddressesMatchingStates(StateFilter ...string) return resp.IPConfigurationStatus, err } + //GetPodOrchestratorContext calls GetPodIpOrchestratorContext API on CNS func (cnsClient *CNSClient) GetPodOrchestratorContext() (map[string]string, error) { var ( resp cns.GetPodContextResponse err error res *http.Response - body bytes.Buffer ) url := cnsClient.connectionURL + cns.GetPodIPOrchestratorContext log.Printf("GetPodIPOrchestratorContext url %v", url) - payload := "" - - err = json.NewEncoder(&body).Encode(payload) + res, err = http.Get(url) if err != nil { - log.Errorf("encoding json failed with %v", err) - return resp.PodContext, err - } - - res, err = http.Post(url, contentTypeJSON, &body) - if err != nil { - log.Errorf("[Azure CNSClient] HTTP Post returned error %v", err.Error()) + log.Errorf("[Azure CNSClient] GetPodIPOrchestratorContext HTTP Get returned error %v", err.Error()) return resp.PodContext, err } defer res.Body.Close() if res.StatusCode != http.StatusOK { - errMsg := fmt.Sprintf("[Azure CNSClient] GetPodContext invalid http status code: %v", res.StatusCode) + errMsg := fmt.Sprintf("[Azure CNSClient] GetPodIPOrchestratorContext invalid http status code: %v", res.StatusCode) log.Errorf(errMsg) return resp.PodContext, fmt.Errorf(errMsg) } diff --git a/cns/cnsclient/cnsclient_test.go b/cns/cnsclient/cnsclient_test.go index d030926f46..2790a60c6e 100644 --- a/cns/cnsclient/cnsclient_test.go +++ b/cns/cnsclient/cnsclient_test.go @@ -11,6 +11,7 @@ import ( "reflect" "strconv" "testing" + "github.com/Azure/azure-container-networking/cns" "github.com/Azure/azure-container-networking/cns/common" "github.com/Azure/azure-container-networking/cns/fakes" @@ -238,21 +239,47 @@ func TestCNSClientRequestAndRelease(t *testing.T) { if ipaddresses[0].IPAddress != desiredIpAddress && ipaddresses[0].State != cns.Allocated { t.Fatalf("Available IP address does not match expected, address state: %+v", ipaddresses) } - fmt.Println(ipaddresses) + + t.Log(ipaddresses) + + // release requested IP address, expect success + err = cnsClient.ReleaseIPAddress(orchestratorContext) + if err != nil { + t.Fatalf("Expected to not fail when releasing IP reservation found with context: %+v", err) + } +} + +func TestCNSClientPodContextApi(t *testing.T) { + podName := "testpodname" + podNamespace := "testpodnamespace" + desiredIpAddress := "10.0.0.5" + + secondaryIps := make([]string, 0) + secondaryIps = append(secondaryIps, desiredIpAddress) + cnsClient, _ := InitCnsClient("") + + addTestStateToRestServer(t, secondaryIps) + + podInfo := cns.KubernetesPodInfo{PodName: podName, PodNamespace: podNamespace} + orchestratorContext, err := json.Marshal(podInfo) + if err != nil { + t.Fatal(err) + } + + // request IP address + _, err = cnsClient.RequestIPAddress(orchestratorContext) + if err != nil { + t.Fatalf("get IP from CNS failed with %+v", err) + } //test for pod ip by orch context map podcontext, err := cnsClient.GetPodOrchestratorContext() if err != nil { - t.Fatalf("Get pod ip by orchestrator context failed %+v", err) + t.Errorf("Get pod ip by orchestrator context failed: %+v", err) } if len(podcontext) < 1 { - t.Fatalf("Error in getting podcontext %+v", podcontext) + t.Errorf("Expected atleast 1 entry in map for podcontext: %+v", podcontext) } - fmt.Println(podcontext) - // release requested IP address, expect success - err = cnsClient.ReleaseIPAddress(orchestratorContext) - if err != nil { - t.Fatalf("Expected to not fail when releasing IP reservation found with context: %+v", err) - } + t.Log(podcontext) } diff --git a/cns/restserver/ipam.go b/cns/restserver/ipam.go index 3e61b544c7..afe93444ef 100644 --- a/cns/restserver/ipam.go +++ b/cns/restserver/ipam.go @@ -100,33 +100,33 @@ func (service *HTTPRestService) MarkIPAsPendingRelease(totalIpsToRelease int) (m defer service.Unlock() for uuid, existingIpConfig := range service.PodIPConfigState { - if existingIpConfig.State == cns.PendingProgramming { - updatedIpConfig, err := service.updateIPConfigState(uuid, cns.PendingRelease, existingIpConfig.OrchestratorContext) - if err != nil { - return nil, err - } + if existingIpConfig.State == cns.PendingProgramming { + updatedIpConfig, err := service.updateIPConfigState(uuid, cns.PendingRelease, existingIpConfig.OrchestratorContext) + if err != nil { + return nil, err + } - pendingReleasedIps[uuid] = updatedIpConfig + pendingReleasedIps[uuid] = updatedIpConfig if len(pendingReleasedIps) == totalIpsToRelease { return pendingReleasedIps, nil } - } + } } - - // if not all expected IPs are set to PendingRelease, then check the Available IPs + + // if not all expected IPs are set to PendingRelease, then check the Available IPs for uuid, existingIpConfig := range service.PodIPConfigState { - if existingIpConfig.State == cns.Available { - updatedIpConfig, err := service.updateIPConfigState(uuid, cns.PendingRelease, existingIpConfig.OrchestratorContext) - if err != nil { - return nil, err - } - - pendingReleasedIps[uuid] = updatedIpConfig - + if existingIpConfig.State == cns.Available { + updatedIpConfig, err := service.updateIPConfigState(uuid, cns.PendingRelease, existingIpConfig.OrchestratorContext) + if err != nil { + return nil, err + } + + pendingReleasedIps[uuid] = updatedIpConfig + if len(pendingReleasedIps) == totalIpsToRelease { return pendingReleasedIps, nil - } - } + } + } } logger.Printf("[MarkIPAsPendingRelease] Set total ips to PendingRelease %d, expected %d", len(pendingReleasedIps), totalIpsToRelease) @@ -140,9 +140,9 @@ func (service *HTTPRestService) updateIPConfigState(ipId string, updatedState st ipConfig.OrchestratorContext = orchestratorContext service.PodIPConfigState[ipId] = ipConfig return ipConfig, nil - } - - return cns.IPConfigurationStatus{}, fmt.Errorf("[updateIPConfigState] Failed to update state %s for the IPConfig. ID %s not found PodIPConfigState", updatedState, ipId) + } + + return cns.IPConfigurationStatus{}, fmt.Errorf("[updateIPConfigState] Failed to update state %s for the IPConfig. ID %s not found PodIPConfigState", updatedState, ipId) } // MarkIpsAsAvailableUntransacted will update pending programming IPs to available if NMAgent side's programmed nc version keep up with nc version. @@ -183,7 +183,6 @@ func (service *HTTPRestService) GetPodIPConfigState() map[string]cns.IPConfigura func (service *HTTPRestService) getPodIPIDByOrchestratorContexthandler(w http.ResponseWriter, r *http.Request) { var ( - req string resp cns.GetPodContextResponse statusCode int returnMessage string @@ -202,12 +201,6 @@ func (service *HTTPRestService) getPodIPIDByOrchestratorContexthandler(w http.Re logger.Response(service.Name, resp, resp.Response.ReturnCode, ReturnCodeToString(resp.Response.ReturnCode), err) }() - err = service.Listener.Decode(w, r, &req) - if err != nil { - returnMessage = err.Error() - return - } - resp.PodContext = service.GetPodIPIDByOrchestratorContext() return @@ -386,7 +379,7 @@ func (service *HTTPRestService) MarkExistingIPsAsPending(pendingIPIDs []string) return fmt.Errorf("Failed to mark IP [%v] as pending, currently allocated", id) } - logger.Printf("[MarkExistingIPsAsPending]: Marking IP [%+v] to PendingRelease", ipconfig) + logger.Printf("[MarkExistingIPsAsPending]: Marking IP [%+v] to PendingRelease", ipconfig) ipconfig.State = cns.PendingRelease service.PodIPConfigState[id] = ipconfig } else {