diff --git a/cns/NetworkContainerContract.go b/cns/NetworkContainerContract.go index 4f0940e9a6..3fdee87753 100644 --- a/cns/NetworkContainerContract.go +++ b/cns/NetworkContainerContract.go @@ -22,6 +22,7 @@ const ( DetachContainerFromNetwork = "/network/detachcontainerfromnetwork" RequestIPConfig = "/network/requestipconfig" ReleaseIPConfig = "/network/releaseipconfig" + GetIPAddresses = "/debug/getipaddresses" ) // NetworkContainer Prefixes @@ -197,16 +198,6 @@ type GetNetworkContainerResponse struct { AllowNCToHostCommunication bool } -type GetIPConfigRequest struct { - DesiredIPAddress string - OrchestratorContext json.RawMessage -} - -type GetIPConfigResponse struct { - PodIpInfo PodIpInfo - Response Response -} - // DeleteNetworkContainerRequest specifies the details about the request to delete a specifc network container. type PodIpInfo struct { PodIPConfig IPSubnet @@ -221,6 +212,35 @@ type HostIPInfo struct { Subnet string } +type IPConfigRequest struct { + DesiredIPAddress string + OrchestratorContext json.RawMessage +} + +// IPConfigResponse is used in CNS IPAM mode as a response to CNI ADD +type IPConfigResponse struct { + PodIpInfo PodIpInfo + Response Response +} + +// GetIPAddressesRequest is used in CNS IPAM mode to get the states of IPConfigs +// The IPConfigStateFilter is a slice of IP's to fetch from CNS that match those states +type GetIPAddressesRequest struct { + IPConfigStateFilter []string +} + +// GetIPAddressStateResponse is used in CNS IPAM mode as a response to get IP address state +type GetIPAddressStateResponse struct { + IPAddresses []IPAddressState + Response Response +} + +// IPAddressState Only used in the GetIPConfig API to return IP's that match a filter +type IPAddressState struct { + IPAddress string + State string +} + // DeleteNetworkContainerRequest specifies the details about the request to delete a specifc network container. type DeleteNetworkContainerRequest struct { NetworkContainerid string diff --git a/cns/cnsclient/cli.go b/cns/cnsclient/cli.go new file mode 100644 index 0000000000..7aa314fe36 --- /dev/null +++ b/cns/cnsclient/cli.go @@ -0,0 +1,89 @@ +package cnsclient + +import ( + "fmt" + "sort" + "strings" + + "github.com/Azure/azure-container-networking/cns" +) + +const ( + getCmdArg = "get" + getAvailableArg = "Available" + getAllocatedArg = "Allocated" + getAllArg = "All" + getPendingReleaseArg = "PendingRelease" + + releaseArg = "release" + + eth0InterfaceName = "eth0" + azure0InterfaceName = "azure0" + + defaultCNSURl = "http://localhost:10090" +) + +var ( + availableCmds = []string{ + getCmdArg, + } + + getFlags = []string{ + getAvailableArg, + getAllocatedArg, + getAllocatedArg, + } +) + +func HandleCNSClientCommands(cmd, arg string) error { + cnsClient, err := InitCnsClient(defaultCNSURl) + if err != nil { + return err + } + + switch { + case strings.EqualFold(getCmdArg, cmd): + return getCmd(cnsClient, arg) + default: + return fmt.Errorf("No debug cmd supplied, options are: %v", getCmdArg) + } +} + +func getCmd(client *CNSClient, arg string) error { + var states []string + + switch arg { + case cns.Available: + states = append(states, cns.Available) + + case cns.Allocated: + states = append(states, cns.Allocated) + + case cns.PendingRelease: + states = append(states, cns.PendingRelease) + + default: + states = append(states, cns.Allocated) + states = append(states, cns.Available) + states = append(states, cns.PendingRelease) + } + + addr, err := client.GetIPAddressesMatchingStates(states...) + if err != nil { + return err + } + + printIPAddresses(addr) + return nil +} + +// Sort the addresses based on IP, then write to stdout +func printIPAddresses(addrSlice []cns.IPAddressState) { + sort.Slice(addrSlice, func(i, j int) bool { + return addrSlice[i].IPAddress < addrSlice[j].IPAddress + }) + + for _, addr := range addrSlice { + fmt.Printf("%+v\n", addr) + } +} diff --git a/cns/cnsclient/cnsclient.go b/cns/cnsclient/cnsclient.go index 65b74feb27..caf0474e51 100644 --- a/cns/cnsclient/cnsclient.go +++ b/cns/cnsclient/cnsclient.go @@ -209,11 +209,11 @@ func (cnsClient *CNSClient) DeleteHostNCApipaEndpoint(networkContainerID string) } // RequestIPAddress calls the requestIPAddress in CNS -func (cnsClient *CNSClient) RequestIPAddress(orchestratorContext []byte) (*cns.GetIPConfigResponse, error) { +func (cnsClient *CNSClient) RequestIPAddress(orchestratorContext []byte) (*cns.IPConfigResponse, error) { var ( err error res *http.Response - response *cns.GetIPConfigResponse + response *cns.IPConfigResponse ) defer func() { @@ -227,7 +227,7 @@ func (cnsClient *CNSClient) RequestIPAddress(orchestratorContext []byte) (*cns.G httpc := &http.Client{} url := cnsClient.connectionURL + cns.RequestIPConfig - payload := &cns.GetIPConfigRequest{ + payload := &cns.IPConfigRequest{ OrchestratorContext: orchestratorContext, } @@ -277,7 +277,7 @@ func (cnsClient *CNSClient) ReleaseIPAddress(orchestratorContext []byte) error { url := cnsClient.connectionURL + cns.ReleaseIPConfig log.Printf("ReleaseIPAddress url %v", url) - payload := &cns.GetIPConfigRequest{ + payload := &cns.IPConfigRequest{ OrchestratorContext: orchestratorContext, } @@ -316,3 +316,58 @@ func (cnsClient *CNSClient) ReleaseIPAddress(orchestratorContext []byte) error { return err } + +// GetIPAddressesWithStates takes a variadic number of string parameters, to get all IP Addresses matching a number of states +// usage GetIPAddressesWithStates(cns.Available, cns.Allocated) +func (cnsClient *CNSClient) GetIPAddressesMatchingStates(StateFilter ...string) ([]cns.IPAddressState, error) { + var ( + resp cns.GetIPAddressStateResponse + err error + res *http.Response + body bytes.Buffer + ) + + if len(StateFilter) == 0 { + return []cns.IPAddressState{}, nil + } + + url := cnsClient.connectionURL + cns.GetIPAddresses + log.Printf("GetIPAddressesMatchingStates url %v", url) + + payload := &cns.GetIPAddressesRequest{ + IPConfigStateFilter: StateFilter, + } + + err = json.NewEncoder(&body).Encode(payload) + if err != nil { + log.Errorf("encoding json failed with %v", err) + return resp.IPAddresses, err + } + + res, err = http.Post(url, contentTypeJSON, &body) + if err != nil { + log.Errorf("[Azure CNSClient] HTTP Post returned error %v", err.Error()) + return resp.IPAddresses, err + } + + defer res.Body.Close() + + if res.StatusCode != http.StatusOK { + errMsg := fmt.Sprintf("[Azure CNSClient] GetIPAddressesMatchingStates invalid http status code: %v", res.StatusCode) + log.Errorf(errMsg) + return resp.IPAddresses, fmt.Errorf(errMsg) + } + + err = json.NewDecoder(res.Body).Decode(&resp) + if err != nil { + log.Errorf("[Azure CNSClient] Error received while parsing GetIPAddressesMatchingStates response resp:%v err:%v", res.Body, err.Error()) + return resp.IPAddresses, err + } + + if resp.Response.ReturnCode != 0 { + log.Errorf("[Azure CNSClient] GetIPAddressesMatchingStates received error response :%v", resp.Response.Message) + return resp.IPAddresses, fmt.Errorf(resp.Response.Message) + } + + return resp.IPAddresses, err +} diff --git a/cns/cnsclient/cnsclient_test.go b/cns/cnsclient/cnsclient_test.go index 372f134e48..aeae417033 100644 --- a/cns/cnsclient/cnsclient_test.go +++ b/cns/cnsclient/cnsclient_test.go @@ -71,7 +71,7 @@ func addTestStateToRestServer(t *testing.T, secondaryIps []string) { } } -func getIPNetFromResponse(resp *cns.GetIPConfigResponse) (net.IPNet, error) { +func getIPNetFromResponse(resp *cns.IPConfigResponse) (net.IPNet, error) { var ( resultIPnet net.IPNet err error @@ -222,4 +222,17 @@ func TestCNSClientRequestAndRelease(t *testing.T) { if err != nil { t.Fatalf("Expected to not fail when releasing IP reservation found with context: %+v", err) } + + ipaddresses, err := cnsClient.GetIPAddressesMatchingStates(cns.Available) + if err != nil { + t.Fatalf("Get allocated IP addresses failed %+v", err) + } + + if len(ipaddresses) != 1 { + t.Fatalf("Number of available IP addresses expected to be 1, actual %+v", ipaddresses) + } + + if ipaddresses[0].IPAddress != desiredIpAddress && ipaddresses[0].State != cns.Available { + t.Fatalf("Available IP address does not match expected, address state: %+v", ipaddresses) + } } diff --git a/cns/restserver/internalapi.go b/cns/restserver/internalapi.go index 645f28b27a..d9b2619668 100644 --- a/cns/restserver/internalapi.go +++ b/cns/restserver/internalapi.go @@ -177,7 +177,7 @@ func (service *HTTPRestService) ReconcileNCState(ncRequest *cns.CreateNetworkCon } jsonContext, _ := json.Marshal(kubernetesPodInfo) - ipconfigRequest := cns.GetIPConfigRequest{ + ipconfigRequest := cns.IPConfigRequest{ DesiredIPAddress: secIpConfig.IPAddress, OrchestratorContext: jsonContext, } diff --git a/cns/restserver/ipam.go b/cns/restserver/ipam.go index 26b52799af..cdcf20f5b9 100644 --- a/cns/restserver/ipam.go +++ b/cns/restserver/ipam.go @@ -17,7 +17,7 @@ import ( func (service *HTTPRestService) requestIPConfigHandler(w http.ResponseWriter, r *http.Request) { var ( err error - ipconfigRequest cns.GetIPConfigRequest + ipconfigRequest cns.IPConfigRequest podIpInfo cns.PodIpInfo returnCode int returnMessage string @@ -43,7 +43,7 @@ func (service *HTTPRestService) requestIPConfigHandler(w http.ResponseWriter, r Message: returnMessage, } - reserveResp := &cns.GetIPConfigResponse{ + reserveResp := &cns.IPConfigResponse{ Response: resp, } reserveResp.PodIpInfo = podIpInfo @@ -54,20 +54,14 @@ func (service *HTTPRestService) requestIPConfigHandler(w http.ResponseWriter, r func (service *HTTPRestService) releaseIPConfigHandler(w http.ResponseWriter, r *http.Request) { var ( - req cns.GetIPConfigRequest + req cns.IPConfigRequest statusCode int returnMessage string + err error ) statusCode = UnexpectedError - err := service.Listener.Decode(w, r, &req) - logger.Request(service.Name, &req, err) - if err != nil { - returnMessage = err.Error() - return - } - defer func() { resp := cns.Response{} @@ -80,6 +74,13 @@ func (service *HTTPRestService) releaseIPConfigHandler(w http.ResponseWriter, r logger.Response(service.Name, resp, resp.ReturnCode, ReturnCodeToString(resp.ReturnCode), err) }() + err = service.Listener.Decode(w, r, &req) + logger.Request(service.Name, &req, err) + if err != nil { + returnMessage = err.Error() + return + } + podInfo, statusCode, returnMessage := service.validateIpConfigRequest(req) if err = service.releaseIPConfig(podInfo); err != nil { @@ -127,6 +128,74 @@ func (service *HTTPRestService) GetPendingProgramIPConfigs() []cns.IPConfigurati }) } +func (service *HTTPRestService) getIPAddressesHandler(w http.ResponseWriter, r *http.Request) { + var ( + req cns.GetIPAddressesRequest + resp cns.GetIPAddressStateResponse + statusCode int + returnMessage string + err error + ) + + statusCode = UnexpectedError + + defer func() { + if err != nil { + resp.Response.ReturnCode = statusCode + resp.Response.Message = returnMessage + } + + err = service.Listener.Encode(w, &resp) + logger.Response(service.Name, resp, resp.Response.ReturnCode, ReturnCodeToString(resp.Response.ReturnCode), err) + }() + + err = service.Listener.Decode(w, r, &req) + if err != nil { + returnMessage = err.Error() + return + } + + filterFunc := func(ipconfig cns.IPConfigurationStatus, states []string) bool { + for _, state := range states { + if ipconfig.State == state { + return true + } + } + return false + } + + // Get all IPConfigs matching a state, and append to a slice of IPAddressState + resp.IPAddresses = filterIPConfigsMatchingState(service.PodIPConfigState, req.IPConfigStateFilter, filterFunc) + + return +} + +// filter the ipconfigs in CNS matching a state (Available, Allocated, etc.) and return in a slice +func filterIPConfigsMatchingState(toBeAdded map[string]cns.IPConfigurationStatus, states []string, f func(cns.IPConfigurationStatus, []string) bool) []cns.IPAddressState { + vsf := make([]cns.IPAddressState, 0) + for _, v := range toBeAdded { + if f(v, states) { + ip := cns.IPAddressState{ + IPAddress: v.IPAddress, + State: v.State, + } + vsf = append(vsf, ip) + } + } + return vsf +} + +// filter ipconfigs based on predicate +func filterIPConfigs(toBeAdded map[string]cns.IPConfigurationStatus, f func(cns.IPConfigurationStatus) bool) []cns.IPConfigurationStatus { + vsf := make([]cns.IPConfigurationStatus, 0) + for _, v := range toBeAdded { + if f(v) { + vsf = append(vsf, v) + } + } + return vsf +} + func (service *HTTPRestService) GetAllocatedIPConfigs() []cns.IPConfigurationStatus { service.RLock() defer service.RUnlock() @@ -300,7 +369,7 @@ func (service *HTTPRestService) AllocateAnyAvailableIPConfig(podInfo cns.Kuberne } // If IPConfig is already allocated for pod, it returns that else it returns one of the available ipconfigs. -func requestIPConfigHelper(service *HTTPRestService, req cns.GetIPConfigRequest) (cns.PodIpInfo, error) { +func requestIPConfigHelper(service *HTTPRestService, req cns.IPConfigRequest) (cns.PodIpInfo, error) { var ( podInfo cns.KubernetesPodInfo podIpInfo cns.PodIpInfo diff --git a/cns/restserver/ipam_test.go b/cns/restserver/ipam_test.go index f1a280eea3..b6944978ef 100644 --- a/cns/restserver/ipam_test.go +++ b/cns/restserver/ipam_test.go @@ -66,7 +66,7 @@ func NewPodState(ipaddress string, prefixLength uint8, id, ncid, state string) c } } -func requestIpAddressAndGetState(t *testing.T, req cns.GetIPConfigRequest) (cns.IPConfigurationStatus, error) { +func requestIpAddressAndGetState(t *testing.T, req cns.IPConfigRequest) (cns.IPConfigurationStatus, error) { var ( podInfo cns.KubernetesPodInfo ipState cns.IPConfigurationStatus @@ -172,7 +172,7 @@ func TestIPAMGetAvailableIPConfig(t *testing.T) { } UpdatePodIpConfigState(t, svc, ipconfigs) - req := cns.GetIPConfigRequest{} + req := cns.IPConfigRequest{} b, _ := json.Marshal(testPod1Info) req.OrchestratorContext = b @@ -207,7 +207,7 @@ func TestIPAMGetNextAvailableIPConfig(t *testing.T) { t.Fatalf("Expected to not fail adding IP's to state: %+v", err) } - req := cns.GetIPConfigRequest{} + req := cns.IPConfigRequest{} b, _ := json.Marshal(testPod2Info) req.OrchestratorContext = b @@ -236,7 +236,7 @@ func TestIPAMGetAlreadyAllocatedIPConfigForSamePod(t *testing.T) { t.Fatalf("Expected to not fail adding IP's to state: %+v", err) } - req := cns.GetIPConfigRequest{} + req := cns.IPConfigRequest{} b, _ := json.Marshal(testPod1Info) req.OrchestratorContext = b @@ -266,7 +266,7 @@ func TestIPAMAttemptToRequestIPNotFoundInPool(t *testing.T) { t.Fatalf("Expected to not fail adding IP's to state: %+v", err) } - req := cns.GetIPConfigRequest{} + req := cns.IPConfigRequest{} b, _ := json.Marshal(testPod2Info) req.OrchestratorContext = b req.DesiredIPAddress = testIP2 @@ -291,7 +291,7 @@ func TestIPAMGetDesiredIPConfigWithSpecfiedIP(t *testing.T) { t.Fatalf("Expected to not fail adding IP's to state: %+v", err) } - req := cns.GetIPConfigRequest{} + req := cns.IPConfigRequest{} b, _ := json.Marshal(testPod1Info) req.OrchestratorContext = b req.DesiredIPAddress = testIP1 @@ -323,7 +323,7 @@ func TestIPAMFailToGetDesiredIPConfigWithAlreadyAllocatedSpecfiedIP(t *testing.T } // request the already allocated ip with a new context - req := cns.GetIPConfigRequest{} + req := cns.IPConfigRequest{} b, _ := json.Marshal(testPod2Info) req.OrchestratorContext = b req.DesiredIPAddress = testIP1 @@ -351,7 +351,7 @@ func TestIPAMFailToGetIPWhenAllIPsAreAllocated(t *testing.T) { } // request the already allocated ip with a new context - req := cns.GetIPConfigRequest{} + req := cns.IPConfigRequest{} b, _ := json.Marshal(testPod3Info) req.OrchestratorContext = b @@ -382,7 +382,7 @@ func TestIPAMRequestThenReleaseThenRequestAgain(t *testing.T) { desiredIpAddress := testIP1 // Use TestPodInfo2 to request TestIP1, which has already been allocated - req := cns.GetIPConfigRequest{} + req := cns.IPConfigRequest{} b, _ := json.Marshal(testPod2Info) req.OrchestratorContext = b req.DesiredIPAddress = desiredIpAddress @@ -399,7 +399,7 @@ func TestIPAMRequestThenReleaseThenRequestAgain(t *testing.T) { } // Rerequest - req = cns.GetIPConfigRequest{} + req = cns.IPConfigRequest{} b, _ = json.Marshal(testPod2Info) req.OrchestratorContext = b req.DesiredIPAddress = desiredIpAddress @@ -491,7 +491,7 @@ func TestAvailableIPConfigs(t *testing.T) { allocatedIps := svc.GetAllocatedIPConfigs() validateIpState(t, allocatedIps, desiredAllocatedIpConfigs) - req := cns.GetIPConfigRequest{} + req := cns.IPConfigRequest{} b, _ := json.Marshal(testPod1Info) req.OrchestratorContext = b req.DesiredIPAddress = state1.IPAddress diff --git a/cns/restserver/restserver.go b/cns/restserver/restserver.go index cce2494743..f92d5414f5 100644 --- a/cns/restserver/restserver.go +++ b/cns/restserver/restserver.go @@ -176,6 +176,7 @@ func (service *HTTPRestService) Start(config *common.ServiceConfig) error { listener.AddHandler(cns.UnpublishNetworkContainer, service.unpublishNetworkContainer) listener.AddHandler(cns.RequestIPConfig, service.requestIPConfigHandler) listener.AddHandler(cns.ReleaseIPConfig, service.releaseIPConfigHandler) + listener.AddHandler(cns.GetIPAddresses, service.getIPAddressesHandler) // handlers for v0.2 listener.AddHandler(cns.V2Prefix+cns.SetEnvironmentPath, service.setEnvironment) diff --git a/cns/restserver/util.go b/cns/restserver/util.go index 645aa06d62..30bf8da94d 100644 --- a/cns/restserver/util.go +++ b/cns/restserver/util.go @@ -625,7 +625,7 @@ func (service *HTTPRestService) SendNCSnapShotPeriodically(ncSnapshotIntervalInM } } -func (service *HTTPRestService) validateIpConfigRequest(ipConfigRequest cns.GetIPConfigRequest) (cns.KubernetesPodInfo, int, string) { +func (service *HTTPRestService) validateIpConfigRequest(ipConfigRequest cns.IPConfigRequest) (cns.KubernetesPodInfo, int, string) { var podInfo cns.KubernetesPodInfo if service.state.OrchestratorType != cns.KubernetesCRD { diff --git a/cns/service/main.go b/cns/service/main.go index 330ebf1391..f4c757f42d 100644 --- a/cns/service/main.go +++ b/cns/service/main.go @@ -23,6 +23,7 @@ import ( "github.com/Azure/azure-container-networking/cnm/ipam" "github.com/Azure/azure-container-networking/cnm/network" "github.com/Azure/azure-container-networking/cns" + "github.com/Azure/azure-container-networking/cns/cnsclient" "github.com/Azure/azure-container-networking/cns/common" "github.com/Azure/azure-container-networking/cns/configuration" "github.com/Azure/azure-container-networking/cns/hnsclient" @@ -222,6 +223,20 @@ var args = acn.ArgumentList{ Type: "bool", DefaultValue: false, }, + { + Name: acn.OptDebugCmd, + Shorthand: acn.OptDebugCmdAlias, + Description: "Debug flag to retrieve IPconfigs, available values: allocated, available, all", + Type: "string", + DefaultValue: "", + }, + { + Name: acn.OptDebugArg, + Shorthand: acn.OptDebugArgAlias, + Description: "Argument flag to be paired with the 'debugcmd' flag.", + Type: "string", + DefaultValue: "", + }, } // Prints description and version information. @@ -294,6 +309,8 @@ func main() { privateEndpoint := acn.GetArg(acn.OptPrivateEndpoint).(string) infravnet := acn.GetArg(acn.OptInfrastructureNetworkID).(string) nodeID := acn.GetArg(acn.OptNodeID).(string) + clientDebugCmd := acn.GetArg(acn.OptDebugCmd).(string) + clientDebugArg := acn.GetArg(acn.OptDebugArg).(string) if vers { printVersion() @@ -314,6 +331,15 @@ func main() { // Create logging provider. logger.InitLogger(name, logLevel, logTarget, logDirectory) + if clientDebugCmd != "" { + err := cnsclient.HandleCNSClientCommands(clientDebugCmd, clientDebugArg) + if err != nil { + fmt.Println(err) + os.Exit(1) + } + os.Exit(0) + } + if !telemetryEnabled { logger.Errorf("[Azure CNS] Cannot disable telemetry via cmdline. Update cns_config.json to disable telemetry.") } diff --git a/common/config.go b/common/config.go index 1239a04e96..024349a4ce 100644 --- a/common/config.go +++ b/common/config.go @@ -114,4 +114,12 @@ const ( // Managed mode OptManaged = "managed" OptManagedAlias = "m" + + // Client mode, cmd + OptDebugCmd = "debugcmd" + OptDebugCmdAlias = "cmd" + + // Client mode, args for cmd + OptDebugArg = "debugarg" + OptDebugArgAlias = "darg" )