diff --git a/cns/client/client_test.go b/cns/client/client_test.go index a6ea4793ea..61c6436dab 100644 --- a/cns/client/client_test.go +++ b/cns/client/client_test.go @@ -168,7 +168,7 @@ func TestMain(m *testing.M) { logger.InitLogger(logName, 0, 0, tmpLogDir+"/") config := common.ServiceConfig{} - httpRestService, err := restserver.NewHTTPRestService(&config, fakes.NewFakeImdsClient(), fakes.NewFakeNMAgentClient()) + httpRestService, err := restserver.NewHTTPRestService(&config, &fakes.WireserverClientFake{}, fakes.NewFakeNMAgentClient()) svc = httpRestService.(*restserver.HTTPRestService) svc.Name = "cns-test-server" fakeNNC := v1alpha.NodeNetworkConfig{ diff --git a/cns/dockerclient/dockerclient.go b/cns/dockerclient/dockerclient.go index b28c108084..406e58245b 100644 --- a/cns/dockerclient/dockerclient.go +++ b/cns/dockerclient/dockerclient.go @@ -5,14 +5,16 @@ package dockerclient import ( "bytes" + "context" "encoding/json" "fmt" "net/http" - "github.com/Azure/azure-container-networking/cns/imdsclient" "github.com/Azure/azure-container-networking/cns/logger" + "github.com/Azure/azure-container-networking/cns/wireserver" "github.com/Azure/azure-container-networking/common" "github.com/Azure/azure-container-networking/platform" + "github.com/pkg/errors" ) const ( @@ -22,34 +24,30 @@ const ( bridgeMode = "bridge" ) -// DockerClient specifies a client to connect to docker. -type DockerClient struct { - connectionURL string - imdsClient imdsclient.ImdsClientInterface +type interfaceGetter interface { + GetInterfaces(ctx context.Context) (*wireserver.GetInterfacesResult, error) } -// NewDockerClient create a new docker client. -func NewDockerClient(url string) (*DockerClient, error) { - return &DockerClient{ - connectionURL: url, - imdsClient: new(imdsclient.ImdsClient), - }, nil +// Client specifies a client to connect to docker. +type Client struct { + connectionURL string + wscli interfaceGetter } -// NewDefaultDockerClient create a new docker client. -func NewDefaultDockerClient(imdsClient imdsclient.ImdsClientInterface) (*DockerClient, error) { - return &DockerClient{ +// NewDefaultClient create a new docker client. +func NewDefaultClient(wscli interfaceGetter) (*Client, error) { + return &Client{ connectionURL: defaultDockerConnectionURL, - imdsClient: imdsClient, + wscli: wscli, }, nil } // NetworkExists tries to retrieve a network from docker (if it exists). -func (dockerClient *DockerClient) NetworkExists(networkName string) error { +func (c *Client) NetworkExists(networkName string) error { logger.Printf("[Azure CNS] NetworkExists") res, err := http.Get( - dockerClient.connectionURL + inspectNetworkPath + networkName) + c.connectionURL + inspectNetworkPath + networkName) if err != nil { logger.Errorf("[Azure CNS] Error received from http Post for docker network inspect %v %v", networkName, err.Error()) return err @@ -73,7 +71,7 @@ func (dockerClient *DockerClient) NetworkExists(networkName string) error { } // CreateNetwork creates a network using docker network create. -func (dockerClient *DockerClient) CreateNetwork(networkName string, nicInfo *imdsclient.InterfaceInfo, options map[string]interface{}) error { +func (c *Client) CreateNetwork(networkName string, nicInfo *wireserver.InterfaceInfo, options map[string]interface{}) error { logger.Printf("[Azure CNS] CreateNetwork") enableSnat := true @@ -116,7 +114,7 @@ func (dockerClient *DockerClient) CreateNetwork(networkName string, nicInfo *imd } res, err := http.Post( - dockerClient.connectionURL+createNetworkPath, + c.connectionURL+createNetworkPath, common.JsonContent, netConfigJSON) if err != nil { @@ -149,11 +147,11 @@ func (dockerClient *DockerClient) CreateNetwork(networkName string, nicInfo *imd } // DeleteNetwork creates a network using docker network create. -func (dockerClient *DockerClient) DeleteNetwork(networkName string) error { +func (c *Client) DeleteNetwork(networkName string) error { p := platform.NewExecClient() logger.Printf("[Azure CNS] DeleteNetwork") - url := dockerClient.connectionURL + inspectNetworkPath + networkName + url := c.connectionURL + inspectNetworkPath + networkName req, err := http.NewRequest("DELETE", url, nil) if err != nil { logger.Printf("[Azure CNS] Error received while creating http DELETE request for network delete %v %v", networkName, err.Error()) @@ -172,9 +170,13 @@ func (dockerClient *DockerClient) DeleteNetwork(networkName string) error { // network successfully deleted. if res.StatusCode == 204 { - primaryNic, err := dockerClient.imdsClient.GetPrimaryInterfaceInfoFromHost() + res, err := c.wscli.GetInterfaces(context.TODO()) // TODO(rbtr): thread context through this client + if err != nil { + return errors.Wrap(err, "failed to get interfaces from IMDS") + } + primaryNic, err := wireserver.GetPrimaryInterfaceFromResult(res) if err != nil { - return err + return errors.Wrap(err, "failed to get primary interface from IMDS response") } cmd := fmt.Sprintf("iptables -t nat -D POSTROUTING -m iprange ! --dst-range 168.63.129.16 -m addrtype ! --dst-type local ! -d %v -j MASQUERADE", diff --git a/cns/fakes/imdsclientfake.go b/cns/fakes/imdsclientfake.go index 68f1eda639..21fa0200f2 100644 --- a/cns/fakes/imdsclientfake.go +++ b/cns/fakes/imdsclientfake.go @@ -7,52 +7,37 @@ package fakes import ( - "github.com/Azure/azure-container-networking/cns/imdsclient" - "github.com/Azure/azure-container-networking/cns/logger" + "context" + + "github.com/Azure/azure-container-networking/cns/wireserver" ) var ( - HostPrimaryIpTest = "10.0.0.4" - HostSubnetTest = "10.0.0.0/24" + // HostPrimaryIP 10.0.0.4 + HostPrimaryIP = "10.0.0.4" + // HostSubnet 10.0.0.0/24 + HostSubnet = "10.0.0.0/24" ) -// ImdsClient can be used to connect to VM Host agent in Azure. -type ImdsClientTest struct{} - -func NewFakeImdsClient() *ImdsClientTest { - return &ImdsClientTest{} -} - -// GetNetworkContainerInfoFromHost - Mock implementation to return Container version info. -func (imdsClient *ImdsClientTest) GetNetworkContainerInfoFromHost(networkContainerID string, primaryAddress string, authToken string, apiVersion string) (*imdsclient.ContainerVersion, error) { - ret := &imdsclient.ContainerVersion{} - - return ret, nil -} - -// GetPrimaryInterfaceInfoFromHost - Mock implementation to return Host interface info -func (imdsClient *ImdsClientTest) GetPrimaryInterfaceInfoFromHost() (*imdsclient.InterfaceInfo, error) { - logger.Printf("[Azure CNS] GetPrimaryInterfaceInfoFromHost") - - interfaceInfo := &imdsclient.InterfaceInfo{ - Subnet: HostSubnetTest, - PrimaryIP: HostPrimaryIpTest, - } - - return interfaceInfo, nil -} - -// GetPrimaryInterfaceInfoFromMemory - Mock implementation to return host interface info -func (imdsClient *ImdsClientTest) GetPrimaryInterfaceInfoFromMemory() (*imdsclient.InterfaceInfo, error) { - logger.Printf("[Azure CNS] GetPrimaryInterfaceInfoFromMemory") - - return imdsClient.GetPrimaryInterfaceInfoFromHost() -} - -// GetNetworkContainerInfoFromHostWithoutToken - Mock implementation to return host NMAgent NC version -// Set it as 0 which is the same as default initial NC version for testing purpose -func (imdsClient *ImdsClientTest) GetNetworkContainerInfoFromHostWithoutToken() int { - logger.Printf("[Azure CNS] get the NC version from NMAgent") - - return 0 +type WireserverClientFake struct{} + +func (c *WireserverClientFake) GetInterfaces(ctx context.Context) (*wireserver.GetInterfacesResult, error) { + return &wireserver.GetInterfacesResult{ + Interface: []wireserver.Interface{ + { + IsPrimary: true, + IPSubnet: []wireserver.Subnet{ + { + Prefix: HostSubnet, + IPAddress: []wireserver.Address{ + { + Address: HostPrimaryIP, + IsPrimary: true, + }, + }, + }, + }, + }, + }, + }, nil } diff --git a/cns/imdsclient/api.go b/cns/imdsclient/api.go deleted file mode 100644 index d03b2b9d76..0000000000 --- a/cns/imdsclient/api.go +++ /dev/null @@ -1,68 +0,0 @@ -// Copyright 2017 Microsoft. All rights reserved. -// MIT License - -package imdsclient - -import ( - "encoding/xml" -) - -const ( - hostQueryURL = "http://168.63.129.16/machine/plugins?comp=nmagent&type=getinterfaceinfov1" - hostQueryURLForProgrammedVersion = "http://168.63.129.16/machine/plugins/?comp=nmagent&type=NetworkManagement/interfaces/%s/networkContainers/%s/authenticationToken/%s/api-version/%s" -) - -// ImdsClient can be used to connect to VM Host agent in Azure. -type ImdsClient struct { - primaryInterface *InterfaceInfo -} - -// InterfaceInfo specifies the information about an interface as returned by Host Agent. -type InterfaceInfo struct { - Subnet string - Gateway string - IsPrimary bool - PrimaryIP string - SecondaryIPs []string -} - -// Azure host agent XML document format. -type xmlDocument struct { - XMLName xml.Name `xml:"Interfaces"` - Interface []struct { - XMLName xml.Name `xml:"Interface"` - MacAddress string `xml:"MacAddress,attr"` - IsPrimary bool `xml:"IsPrimary,attr"` - - IPSubnet []struct { - XMLName xml.Name `xml:"IPSubnet"` - Prefix string `xml:"Prefix,attr"` - - IPAddress []struct { - XMLName xml.Name `xml:"IPAddress"` - Address string `xml:"Address,attr"` - IsPrimary bool `xml:"IsPrimary,attr"` - } - } - } -} - -type containerVersionJsonResponse struct { - HTTPResponseCode string `json:"httpResponseCode"` - NetworkContainerID string `json:"networkContainerId"` - ProgrammedVersion string `json:"Version"` -} - -// InterfaceInfo specifies the information about an interface as returned by Host Agent. -type ContainerVersion struct { - NetworkContainerID string - ProgrammedVersion string -} - -// An ImdsInterface performs CRUD operations on IP reservations -type ImdsClientInterface interface { - GetNetworkContainerInfoFromHost(networkContainerID string, primaryAddress string, authToken string, apiVersion string) (*ContainerVersion, error) - GetPrimaryInterfaceInfoFromHost() (*InterfaceInfo, error) - GetPrimaryInterfaceInfoFromMemory() (*InterfaceInfo, error) - GetNetworkContainerInfoFromHostWithoutToken() int -} diff --git a/cns/imdsclient/imdsclient.go b/cns/imdsclient/imdsclient.go deleted file mode 100644 index ccfd0fa2d3..0000000000 --- a/cns/imdsclient/imdsclient.go +++ /dev/null @@ -1,174 +0,0 @@ -package imdsclient - -import ( - "bytes" - "encoding/json" - "encoding/xml" - "fmt" - "io" - "math" - "net" - "net/http" - - "github.com/Azure/azure-container-networking/cns/logger" - "github.com/pkg/errors" -) - -var ( - // ErrNoPrimaryInterface indicates the imds respnose does not have a primary interface indicated. - ErrNoPrimaryInterface = errors.New("no primary interface found") - // ErrInsufficientAddressSpace indicates that the CIDR space is too small to include a gateway IP; it is 1 IP. - ErrInsufficientAddressSpace = errors.New("insufficient address space to generate gateway IP") -) - -// GetNetworkContainerInfoFromHost retrieves the programmed version of network container from Host. -func (imdsClient *ImdsClient) GetNetworkContainerInfoFromHost(networkContainerID string, primaryAddress string, authToken string, apiVersion string) (*ContainerVersion, error) { - logger.Printf("[Azure CNS] GetNetworkContainerInfoFromHost") - queryURL := fmt.Sprintf(hostQueryURLForProgrammedVersion, - primaryAddress, networkContainerID, authToken, apiVersion) - - logger.Printf("[Azure CNS] Going to query Azure Host for container version @\n %v\n", queryURL) - jsonResponse, err := http.Get(queryURL) - if err != nil { - return nil, err - } - - defer jsonResponse.Body.Close() - - logger.Printf("[Azure CNS] Response received from Azure Host for NetworkManagement/interfaces: %v", jsonResponse.Body) - - var response containerVersionJsonResponse - err = json.NewDecoder(jsonResponse.Body).Decode(&response) - if err != nil { - return nil, err - } - - ret := &ContainerVersion{ - NetworkContainerID: response.NetworkContainerID, - ProgrammedVersion: response.ProgrammedVersion, - } - - return ret, nil -} - -// GetPrimaryInterfaceInfoFromHost retrieves subnet and gateway of primary NIC from Host. -// TODO(rbtr): this is not a good client contract, we should return the resp. -func (imdsClient *ImdsClient) GetPrimaryInterfaceInfoFromHost() (*InterfaceInfo, error) { - logger.Printf("[Azure CNS] GetPrimaryInterfaceInfoFromHost") - - interfaceInfo := &InterfaceInfo{} - resp, err := http.Get(hostQueryURL) - if err != nil { - return nil, err - } - defer resp.Body.Close() - b, err := io.ReadAll(resp.Body) - if err != nil { - return nil, errors.Wrap(err, "failed to read response body") - } - - logger.Printf("[Azure CNS] Response received from NMAgent for get interface details: %s", string(b)) - - var doc xmlDocument - if err := xml.NewDecoder(bytes.NewReader(b)).Decode(&doc); err != nil { - return nil, errors.Wrap(err, "failed to decode response body") - } - - foundPrimaryInterface := false - - // For each interface. - for _, i := range doc.Interface { - // skip if not primary - if !i.IsPrimary { - continue - } - interfaceInfo.IsPrimary = true - - // Get the first subnet. - for _, s := range i.IPSubnet { - interfaceInfo.Subnet = s.Prefix - gw, err := calculateGatewayIP(s.Prefix) - if err != nil { - return nil, err - } - interfaceInfo.Gateway = gw.String() - for _, ip := range s.IPAddress { - if ip.IsPrimary { - interfaceInfo.PrimaryIP = ip.Address - } - } - - imdsClient.primaryInterface = interfaceInfo - break - } - - foundPrimaryInterface = true - break - } - - if !foundPrimaryInterface { - return nil, ErrNoPrimaryInterface - } - - return interfaceInfo, nil -} - -// calculateGatewayIP parses the passed CIDR string and returns the first IP in the range. -func calculateGatewayIP(cidr string) (net.IP, error) { - _, subnet, err := net.ParseCIDR(cidr) - if err != nil { - return nil, errors.Wrap(err, "received malformed subnet from host") - } - - // check if we have enough address space to calculate a gateway IP - // we need at least 2 IPs (eg the IPv4 mask cannot be greater than 31) - // since the zeroth is reserved and the gateway is the first. - mask, bits := subnet.Mask.Size() - if mask == bits { - return nil, ErrInsufficientAddressSpace - } - - // the subnet IP is the zero base address, so we need to increment it by one to get the gateway. - gw := make([]byte, len(subnet.IP)) - copy(gw, subnet.IP) - for idx := len(gw) - 1; idx >= 0; idx-- { - gw[idx]++ - // net.IP is a binary byte array, check if we have overflowed and need to continue incrementing to the left - // along the arary or if we're done. - // it's like if we have a 9 in base 10, and add 1, it rolls over to 0 so we're not done - we need to move - // left and increment that digit also. - if gw[idx] != 0 { - break - } - } - return gw, nil -} - -// GetPrimaryInterfaceInfoFromMemory retrieves subnet and gateway of primary NIC that is saved in memory. -func (imdsClient *ImdsClient) GetPrimaryInterfaceInfoFromMemory() (*InterfaceInfo, error) { - logger.Printf("[Azure CNS] GetPrimaryInterfaceInfoFromMemory") - - var iface *InterfaceInfo - var err error - if imdsClient.primaryInterface == nil { - logger.Debugf("Azure-CNS] Primary interface in memory does not exist. Will get it from Host.") - iface, err = imdsClient.GetPrimaryInterfaceInfoFromHost() - if err != nil { - logger.Errorf("[Azure-CNS] Unable to retrive primary interface info.") - } else { - logger.Debugf("Azure-CNS] Primary interface received from HOST: %+v.", iface) - } - } else { - iface = imdsClient.primaryInterface - } - - return iface, err -} - -// GetNetworkContainerInfoFromHostWithoutToken is a temp implementation which will be removed once background thread -// updating host version is ready. Return max integer value to regress current AKS scenario -func (imdsClient *ImdsClient) GetNetworkContainerInfoFromHostWithoutToken() int { - logger.Printf("[Azure CNS] GetNMagentVersionFromNMAgent") - - return math.MaxInt64 -} diff --git a/cns/restserver/api.go b/cns/restserver/api.go index 64c8da806c..4408ecefe9 100644 --- a/cns/restserver/api.go +++ b/cns/restserver/api.go @@ -4,6 +4,7 @@ package restserver import ( + "context" "fmt" "io/ioutil" "net" @@ -17,6 +18,7 @@ import ( "github.com/Azure/azure-container-networking/cns/logger" "github.com/Azure/azure-container-networking/cns/nmagentclient" "github.com/Azure/azure-container-networking/cns/types" + "github.com/Azure/azure-container-networking/cns/wireserver" "github.com/Azure/azure-container-networking/common" "github.com/Azure/azure-container-networking/platform" ) @@ -66,7 +68,7 @@ func (service *HTTPRestService) createNetwork(w http.ResponseWriter, r *http.Req logger.Request(service.Name, &req, err) if err != nil { - //nolint:goconst + //nolint:goconst // ignore const string returnMessage = "[Azure CNS] Error. Unable to decode input request." returnCode = types.InvalidParameter } else { @@ -93,7 +95,8 @@ func (service *HTTPRestService) createNetwork(w http.ResponseWriter, r *http.Req logger.Printf("[Azure CNS] Unable to get routing table from node, %+v.", err.Error()) } - nicInfo, err := service.imdsClient.GetPrimaryInterfaceInfoFromHost() + var nicInfo *wireserver.InterfaceInfo + nicInfo, err = service.getPrimaryHostInterface(context.TODO()) if err != nil { returnMessage = fmt.Sprintf("[Azure CNS] Error. GetPrimaryInterfaceInfoFromHost failed %v.", err.Error()) returnCode = types.UnexpectedError @@ -342,7 +345,8 @@ func (service *HTTPRestService) reserveIPAddress(w http.ResponseWriter, r *http. case "POST": ic := service.ipamClient - ifInfo, err := service.imdsClient.GetPrimaryInterfaceInfoFromMemory() + var ifInfo *wireserver.InterfaceInfo + ifInfo, err = service.getPrimaryHostInterface(context.TODO()) if err != nil { returnMessage = fmt.Sprintf("[Azure CNS] Error. GetPrimaryIfaceInfo failed %v", err.Error()) returnCode = types.UnexpectedError @@ -418,7 +422,8 @@ func (service *HTTPRestService) releaseIPAddress(w http.ResponseWriter, r *http. case "POST": ic := service.ipamClient - ifInfo, err := service.imdsClient.GetPrimaryInterfaceInfoFromMemory() + var ifInfo *wireserver.InterfaceInfo + ifInfo, err = service.getPrimaryHostInterface(context.TODO()) if err != nil { returnMessage = fmt.Sprintf("[Azure CNS] Error. GetPrimaryIfaceInfo failed %v", err.Error()) returnCode = types.UnexpectedError @@ -473,8 +478,8 @@ func (service *HTTPRestService) getHostLocalIP(w http.ResponseWriter, r *http.Re case "GET": switch service.state.NetworkType { case "Underlay": - if service.imdsClient != nil { - piface, err := service.imdsClient.GetPrimaryInterfaceInfoFromMemory() + if service.wscli != nil { + piface, err := service.getPrimaryHostInterface(context.TODO()) if err == nil { hostLocalIP = piface.PrimaryIP found = true @@ -526,7 +531,7 @@ func (service *HTTPRestService) getIPAddressUtilization(w http.ResponseWriter, r case "GET": ic := service.ipamClient - ifInfo, err := service.imdsClient.GetPrimaryInterfaceInfoFromMemory() + ifInfo, err := service.getPrimaryHostInterface(context.TODO()) if err != nil { returnMessage = fmt.Sprintf("[Azure CNS] Error. GetPrimaryIfaceInfo failed %v", err.Error()) returnCode = types.UnexpectedError @@ -625,7 +630,7 @@ func (service *HTTPRestService) getUnhealthyIPAddresses(w http.ResponseWriter, r case "GET": ic := service.ipamClient - ifInfo, err := service.imdsClient.GetPrimaryInterfaceInfoFromMemory() + ifInfo, err := service.getPrimaryHostInterface(context.TODO()) if err != nil { returnMessage = fmt.Sprintf("[Azure CNS] Error. GetPrimaryIfaceInfo failed %v", err.Error()) returnCode = types.UnexpectedError diff --git a/cns/restserver/api_test.go b/cns/restserver/api_test.go index e1566918f5..c4cbf7bad8 100644 --- a/cns/restserver/api_test.go +++ b/cns/restserver/api_test.go @@ -914,7 +914,7 @@ func startService() error { return err } - service, err = NewHTTPRestService(&config, fakes.NewFakeImdsClient(), fakes.NewFakeNMAgentClient()) + service, err = NewHTTPRestService(&config, &fakes.WireserverClientFake{}, fakes.NewFakeNMAgentClient()) if err != nil { return err } diff --git a/cns/restserver/ipam.go b/cns/restserver/ipam.go index 1e2aea8982..a0d17a5187 100644 --- a/cns/restserver/ipam.go +++ b/cns/restserver/ipam.go @@ -366,7 +366,7 @@ func (service *HTTPRestService) GetExistingIPConfig(podInfo cns.PodInfo) (cns.Po ipID := service.PodIPIDByPodInterfaceKey[podInfo.Key()] if ipID != "" { if ipState, isExist := service.PodIPConfigState[ipID]; isExist { - err := service.populateIpConfigInfoUntransacted(ipState, &podIpInfo) + err := service.populateIPConfigInfoUntransacted(ipState, &podIpInfo) return podIpInfo, isExist, err } @@ -406,7 +406,7 @@ func (service *HTTPRestService) AllocateDesiredIPConfig(podInfo cns.PodInfo, des } if found { - err := service.populateIpConfigInfoUntransacted(ipConfig, &podIpInfo) + err := service.populateIPConfigInfoUntransacted(ipConfig, &podIpInfo) return podIpInfo, err } } @@ -425,7 +425,7 @@ func (service *HTTPRestService) AllocateAnyAvailableIPConfig(podInfo cns.PodInfo } podIPInfo := cns.PodIpInfo{} - if err := service.populateIpConfigInfoUntransacted(ipState, &podIPInfo); err != nil { + if err := service.populateIPConfigInfoUntransacted(ipState, &podIPInfo); err != nil { return cns.PodIpInfo{}, err } diff --git a/cns/restserver/ipam_test.go b/cns/restserver/ipam_test.go index 9ab3ae20d0..17af33883f 100644 --- a/cns/restserver/ipam_test.go +++ b/cns/restserver/ipam_test.go @@ -35,7 +35,7 @@ var ( func getTestService() *HTTPRestService { var config common.ServiceConfig - httpsvc, _ := NewHTTPRestService(&config, fakes.NewFakeImdsClient(), fakes.NewFakeNMAgentClient()) + httpsvc, _ := NewHTTPRestService(&config, &fakes.WireserverClientFake{}, fakes.NewFakeNMAgentClient()) svc = httpsvc.(*HTTPRestService) svc.IPAMPoolMonitor = &fakes.MonitorFake{} setOrchestratorTypeInternal(cns.KubernetesCRD) @@ -94,12 +94,12 @@ func requestIpAddressAndGetState(t *testing.T, req cns.IPConfigRequest) (cns.IPC t.Fatalf("Pod IP Prefix length is not added as expected ipConfig %+v, expected: %+v", PodIpInfo.PodIPConfig, subnetPrfixLength) } - if reflect.DeepEqual(PodIpInfo.HostPrimaryIPInfo.PrimaryIP, fakes.HostPrimaryIpTest) != true { - t.Fatalf("Host PrimaryIP is not added as expected ipConfig %+v, expected primaryIP: %+v", PodIpInfo.HostPrimaryIPInfo, fakes.HostPrimaryIpTest) + if reflect.DeepEqual(PodIpInfo.HostPrimaryIPInfo.PrimaryIP, fakes.HostPrimaryIP) != true { + t.Fatalf("Host PrimaryIP is not added as expected ipConfig %+v, expected primaryIP: %+v", PodIpInfo.HostPrimaryIPInfo, fakes.HostPrimaryIP) } - if reflect.DeepEqual(PodIpInfo.HostPrimaryIPInfo.Subnet, fakes.HostSubnetTest) != true { - t.Fatalf("Host Subnet is not added as expected ipConfig %+v, expected Host subnet: %+v", PodIpInfo.HostPrimaryIPInfo, fakes.HostSubnetTest) + if reflect.DeepEqual(PodIpInfo.HostPrimaryIPInfo.Subnet, fakes.HostSubnet) != true { + t.Fatalf("Host Subnet is not added as expected ipConfig %+v, expected Host subnet: %+v", PodIpInfo.HostPrimaryIPInfo, fakes.HostSubnet) } // retrieve podinfo from orchestrator context diff --git a/cns/restserver/restserver.go b/cns/restserver/restserver.go index 605a047291..d72e5d6938 100644 --- a/cns/restserver/restserver.go +++ b/cns/restserver/restserver.go @@ -4,13 +4,13 @@ package restserver import ( + "context" "sync" "time" "github.com/Azure/azure-container-networking/cns" "github.com/Azure/azure-container-networking/cns/common" "github.com/Azure/azure-container-networking/cns/dockerclient" - "github.com/Azure/azure-container-networking/cns/imdsclient" "github.com/Azure/azure-container-networking/cns/ipamclient" "github.com/Azure/azure-container-networking/cns/logger" "github.com/Azure/azure-container-networking/cns/networkcontainers" @@ -18,8 +18,10 @@ import ( "github.com/Azure/azure-container-networking/cns/routes" "github.com/Azure/azure-container-networking/cns/types" "github.com/Azure/azure-container-networking/cns/types/bounded" + "github.com/Azure/azure-container-networking/cns/wireserver" acn "github.com/Azure/azure-container-networking/common" "github.com/Azure/azure-container-networking/store" + "github.com/pkg/errors" ) // This file contains the initialization of RestServer. @@ -35,11 +37,15 @@ var ( ncVersionURLs sync.Map ) +type interfaceGetter interface { + GetInterfaces(ctx context.Context) (*wireserver.GetInterfacesResult, error) +} + // HTTPRestService represents http listener for CNS - Container Networking Service. type HTTPRestService struct { *cns.Service - dockerClient *dockerclient.DockerClient - imdsClient imdsclient.ImdsClientInterface + dockerClient *dockerclient.Client + wscli interfaceGetter ipamClient *ipamclient.IpamClient nmagentClient nmagentclient.NMAgentClientInterface networkContainer *networkcontainers.NetworkContainers @@ -92,25 +98,25 @@ type httpRestServiceState struct { Networks map[string]*networkInfo TimeStamp time.Time joinedNetworks map[string]struct{} + primaryInterface *wireserver.InterfaceInfo } type networkInfo struct { NetworkName string - NicInfo *imdsclient.InterfaceInfo + NicInfo *wireserver.InterfaceInfo Options map[string]interface{} } // NewHTTPRestService creates a new HTTP Service object. -func NewHTTPRestService(config *common.ServiceConfig, imdsClientInterface imdsclient.ImdsClientInterface, nmagentClient nmagentclient.NMAgentClientInterface) (cns.HTTPService, error) { +func NewHTTPRestService(config *common.ServiceConfig, wscli interfaceGetter, nmagentClient nmagentclient.NMAgentClientInterface) (cns.HTTPService, error) { service, err := cns.NewService(config.Name, config.Version, config.ChannelMode, config.Store) if err != nil { return nil, err } - imdsClient := imdsClientInterface routingTable := &routes.RoutingTable{} nc := &networkcontainers.NetworkContainers{} - dc, err := dockerclient.NewDefaultDockerClient(imdsClientInterface) + dc, err := dockerclient.NewDefaultClient(wscli) if err != nil { return nil, err } @@ -120,9 +126,20 @@ func NewHTTPRestService(config *common.ServiceConfig, imdsClientInterface imdscl return nil, err } - serviceState := &httpRestServiceState{} - serviceState.Networks = make(map[string]*networkInfo) - serviceState.joinedNetworks = make(map[string]struct{}) + res, err := wscli.GetInterfaces(context.TODO()) // TODO(rbtr): thread context through this client + if err != nil { + return nil, errors.Wrap(err, "failed to get interfaces from IMDS") + } + primaryInterface, err := wireserver.GetPrimaryInterfaceFromResult(res) + if err != nil { + return nil, errors.Wrap(err, "failed to get primary interface from IMDS response") + } + + serviceState := &httpRestServiceState{ + Networks: make(map[string]*networkInfo), + joinedNetworks: make(map[string]struct{}), + primaryInterface: primaryInterface, + } podIPIDByPodInterfaceKey := make(map[string]string) podIPConfigState := make(map[string]cns.IPConfigurationStatus) @@ -131,7 +148,7 @@ func NewHTTPRestService(config *common.ServiceConfig, imdsClientInterface imdscl Service: service, store: service.Service.Store, dockerClient: dc, - imdsClient: imdsClient, + wscli: wscli, ipamClient: ic, nmagentClient: nmagentClient, networkContainer: nc, diff --git a/cns/restserver/util.go b/cns/restserver/util.go index 743fd4082d..167c103e5a 100644 --- a/cns/restserver/util.go +++ b/cns/restserver/util.go @@ -19,9 +19,11 @@ import ( "github.com/Azure/azure-container-networking/cns/networkcontainers" "github.com/Azure/azure-container-networking/cns/nmagentclient" "github.com/Azure/azure-container-networking/cns/types" + "github.com/Azure/azure-container-networking/cns/wireserver" acn "github.com/Azure/azure-container-networking/common" "github.com/Azure/azure-container-networking/platform" "github.com/Azure/azure-container-networking/store" + "github.com/pkg/errors" ) // This file contains the utility/helper functions called by either HTTP APIs or Exported/Internal APIs on HTTPRestService @@ -705,34 +707,46 @@ func (service *HTTPRestService) validateIPConfigRequest( return podInfo, types.Success, "" } -func (service *HTTPRestService) populateIpConfigInfoUntransacted(ipConfigStatus cns.IPConfigurationStatus, podIpInfo *cns.PodIpInfo) error { - var ( - ncStatus containerstatus - exists bool - primaryIpConfiguration cns.IPConfiguration - ) +// getPrimaryHostInterface returns the cached InterfaceInfo, if available, otherwise +// queries the IMDS to get the primary interface info and caches it in the server state +// before returning the result. +func (service *HTTPRestService) getPrimaryHostInterface(ctx context.Context) (*wireserver.InterfaceInfo, error) { + if service.state.primaryInterface == nil { + res, err := service.wscli.GetInterfaces(ctx) + if err != nil { + return nil, errors.Wrap(err, "failed to get interfaces from IMDS") + } + service.state.primaryInterface, err = wireserver.GetPrimaryInterfaceFromResult(res) + if err != nil { + return nil, errors.Wrap(err, "failed to get primary interface from IMDS response") + } + } + return service.state.primaryInterface, nil +} - if ncStatus, exists = service.state.ContainerStatus[ipConfigStatus.NCID]; !exists { +//nolint:gocritic // ignore hugeParam pls +func (service *HTTPRestService) populateIPConfigInfoUntransacted(ipConfigStatus cns.IPConfigurationStatus, podIPInfo *cns.PodIpInfo) error { + ncStatus, exists := service.state.ContainerStatus[ipConfigStatus.NCID] + if !exists { return fmt.Errorf("Failed to get NC Configuration for NcId: %s", ipConfigStatus.NCID) } - primaryIpConfiguration = ncStatus.CreateNetworkContainerRequest.IPConfiguration + primaryIPCfg := ncStatus.CreateNetworkContainerRequest.IPConfiguration - podIpInfo.PodIPConfig = cns.IPSubnet{ + podIPInfo.PodIPConfig = cns.IPSubnet{ IPAddress: ipConfigStatus.IPAddress, - PrefixLength: primaryIpConfiguration.IPSubnet.PrefixLength, + PrefixLength: primaryIPCfg.IPSubnet.PrefixLength, } - podIpInfo.NetworkContainerPrimaryIPConfig = primaryIpConfiguration - - hostInterfaceInfo, err := service.imdsClient.GetPrimaryInterfaceInfoFromMemory() + podIPInfo.NetworkContainerPrimaryIPConfig = primaryIPCfg + primaryHostInterface, err := service.getPrimaryHostInterface(context.TODO()) if err != nil { - return fmt.Errorf("Failed to get the HostInterfaceInfo %s", err) + return err } - podIpInfo.HostPrimaryIPInfo.PrimaryIP = hostInterfaceInfo.PrimaryIP - podIpInfo.HostPrimaryIPInfo.Subnet = hostInterfaceInfo.Subnet - podIpInfo.HostPrimaryIPInfo.Gateway = hostInterfaceInfo.Gateway + podIPInfo.HostPrimaryIPInfo.PrimaryIP = primaryHostInterface.PrimaryIP + podIPInfo.HostPrimaryIPInfo.Subnet = primaryHostInterface.Subnet + podIPInfo.HostPrimaryIPInfo.Gateway = primaryHostInterface.Gateway return nil } diff --git a/cns/service/main.go b/cns/service/main.go index 2db6bbf08c..249266f9ff 100644 --- a/cns/service/main.go +++ b/cns/service/main.go @@ -27,7 +27,6 @@ import ( "github.com/Azure/azure-container-networking/cns/common" "github.com/Azure/azure-container-networking/cns/configuration" "github.com/Azure/azure-container-networking/cns/hnsclient" - "github.com/Azure/azure-container-networking/cns/imdsclient" "github.com/Azure/azure-container-networking/cns/ipampool" "github.com/Azure/azure-container-networking/cns/logger" "github.com/Azure/azure-container-networking/cns/multitenantcontroller" @@ -36,6 +35,7 @@ import ( "github.com/Azure/azure-container-networking/cns/restserver" kubecontroller "github.com/Azure/azure-container-networking/cns/singletenantcontroller" cnstypes "github.com/Azure/azure-container-networking/cns/types" + "github.com/Azure/azure-container-networking/cns/wireserver" acn "github.com/Azure/azure-container-networking/common" "github.com/Azure/azure-container-networking/crd" "github.com/Azure/azure-container-networking/crd/nodenetworkconfig" @@ -486,7 +486,8 @@ func main() { return } // Create CNS object. - httpRestService, err := restserver.NewHTTPRestService(&config, new(imdsclient.ImdsClient), nmaclient) + + httpRestService, err := restserver.NewHTTPRestService(&config, &wireserver.Client{HTTPClient: &http.Client{}}, nmaclient) if err != nil { logger.Errorf("Failed to create CNS object, err:%v.\n", err) return diff --git a/cns/wireserver/client.go b/cns/wireserver/client.go new file mode 100644 index 0000000000..43e3c566aa --- /dev/null +++ b/cns/wireserver/client.go @@ -0,0 +1,56 @@ +package wireserver + +import ( + "bytes" + "context" + "encoding/xml" + "io" + "net/http" + + "github.com/Azure/azure-container-networking/cns/logger" + "github.com/pkg/errors" +) + +const hostQueryURL = "http://168.63.129.16/machine/plugins?comp=nmagent&type=getinterfaceinfov1" + +type GetNetworkContainerOpts struct { + NetworkContainerID string + PrimaryAddress string + AuthToken string + APIVersion string +} + +type do interface { + Do(*http.Request) (*http.Response, error) +} + +type Client struct { + HTTPClient do +} + +// GetInterfaces queries interfaces from the wireserver. +func (c *Client) GetInterfaces(ctx context.Context) (*GetInterfacesResult, error) { + logger.Printf("[Azure CNS] GetPrimaryInterfaceInfoFromHost") + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, hostQueryURL, nil) + if err != nil { + return nil, errors.Wrap(err, "failed to construct request") + } + resp, err := c.HTTPClient.Do(req) + if err != nil { + return nil, errors.Wrap(err, "failed to execute request") + } + defer resp.Body.Close() + b, err := io.ReadAll(resp.Body) + if err != nil { + return nil, errors.Wrap(err, "failed to read response body") + } + + logger.Printf("[Azure CNS] Response received from NMAgent for get interface details: %s", string(b)) + + var res GetInterfacesResult + if err := xml.NewDecoder(bytes.NewReader(b)).Decode(&res); err != nil { + return nil, errors.Wrap(err, "failed to decode response body") + } + return &res, nil +} diff --git a/cns/wireserver/net.go b/cns/wireserver/net.go new file mode 100644 index 0000000000..7e96543091 --- /dev/null +++ b/cns/wireserver/net.go @@ -0,0 +1,86 @@ +package wireserver + +import ( + "net" + + "github.com/pkg/errors" +) + +var ( + // ErrNoPrimaryInterface indicates the wireserver respnose does not have a primary interface indicated. + ErrNoPrimaryInterface = errors.New("no primary interface found") + // ErrInsufficientAddressSpace indicates that the CIDR space is too small to include a gateway IP; it is 1 IP. + ErrInsufficientAddressSpace = errors.New("insufficient address space to generate gateway IP") +) + +func GetPrimaryInterfaceFromResult(res *GetInterfacesResult) (*InterfaceInfo, error) { + interfaceInfo := &InterfaceInfo{} + found := false + // For each interface. + for _, i := range res.Interface { + // skip if not primary + if !i.IsPrimary { + continue + } + interfaceInfo.IsPrimary = true + + // skip if no subnets + if len(i.IPSubnet) == 0 { + continue + } + + // get the first subnet + s := i.IPSubnet[0] + interfaceInfo.Subnet = s.Prefix + gw, err := calculateGatewayIP(s.Prefix) + if err != nil { + return nil, err + } + interfaceInfo.Gateway = gw.String() + for _, ip := range s.IPAddress { + if ip.IsPrimary { + interfaceInfo.PrimaryIP = ip.Address + } + } + + found = true + break + } + + if !found { + return nil, ErrNoPrimaryInterface + } + + return interfaceInfo, nil +} + +// calculateGatewayIP parses the passed CIDR string and returns the first IP in the range. +func calculateGatewayIP(cidr string) (net.IP, error) { + _, subnet, err := net.ParseCIDR(cidr) + if err != nil { + return nil, errors.Wrap(err, "received malformed subnet from host") + } + + // check if we have enough address space to calculate a gateway IP + // we need at least 2 IPs (eg the IPv4 mask cannot be greater than 31) + // since the zeroth is reserved and the gateway is the first. + mask, bits := subnet.Mask.Size() + if mask == bits { + return nil, ErrInsufficientAddressSpace + } + + // the subnet IP is the zero base address, so we need to increment it by one to get the gateway. + gw := make([]byte, len(subnet.IP)) + copy(gw, subnet.IP) + for idx := len(gw) - 1; idx >= 0; idx-- { + gw[idx]++ + // net.IP is a binary byte array, check if we have overflowed and need to continue incrementing to the left + // along the arary or if we're done. + // it's like if we have a 9 in base 10, and add 1, it rolls over to 0 so we're not done - we need to move + // left and increment that digit also. + if gw[idx] != 0 { + break + } + } + return gw, nil +} diff --git a/cns/imdsclient/imdsclient_test.go b/cns/wireserver/net_test.go similarity index 98% rename from cns/imdsclient/imdsclient_test.go rename to cns/wireserver/net_test.go index 71dca8fb13..e7488e90a4 100644 --- a/cns/imdsclient/imdsclient_test.go +++ b/cns/wireserver/net_test.go @@ -1,4 +1,4 @@ -package imdsclient +package wireserver import ( "net" diff --git a/cns/imdsclient/testdata/interfaces.xml b/cns/wireserver/testdata/interfaces.xml similarity index 100% rename from cns/imdsclient/testdata/interfaces.xml rename to cns/wireserver/testdata/interfaces.xml diff --git a/cns/wireserver/types.go b/cns/wireserver/types.go new file mode 100644 index 0000000000..f68e7a54f0 --- /dev/null +++ b/cns/wireserver/types.go @@ -0,0 +1,31 @@ +package wireserver + +// InterfaceInfo specifies the information about an interface as returned by Host Agent. +type InterfaceInfo struct { + Subnet string + Gateway string + IsPrimary bool + PrimaryIP string + SecondaryIPs []string +} + +type Address struct { + Address string `xml:"Address,attr"` + IsPrimary bool `xml:"IsPrimary,attr"` +} + +type Subnet struct { + Prefix string `xml:"Prefix,attr"` + IPAddress []Address +} + +type Interface struct { + MacAddress string `xml:"MacAddress,attr"` + IsPrimary bool `xml:"IsPrimary,attr"` + IPSubnet []Subnet +} + +// GetInterfacesResult is the xml mapped response of the getInterfacesQuery +type GetInterfacesResult struct { + Interface []Interface +} diff --git a/cns/wireserver/types_test.go b/cns/wireserver/types_test.go new file mode 100644 index 0000000000..7f7accfc56 --- /dev/null +++ b/cns/wireserver/types_test.go @@ -0,0 +1,38 @@ +package wireserver + +import ( + "bytes" + "encoding/xml" + "os" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestXMLDecode(t *testing.T) { + b, err := os.ReadFile("testdata/interfaces.xml") + require.NoError(t, err) + var resp GetInterfacesResult + require.NoError(t, xml.NewDecoder(bytes.NewReader(b)).Decode(&resp)) + want := GetInterfacesResult{ + Interface: []Interface{ + { + MacAddress: "002248263DBD", + IsPrimary: true, + IPSubnet: []Subnet{ + { + Prefix: "10.240.0.0/16", + IPAddress: []Address{ + { + Address: "10.240.0.4", + IsPrimary: true, + }, + }, + }, + }, + }, + }, + } + assert.Equal(t, want, resp) +}