Cns change cni interfacefor secondary ips

cns/cnsclient/cnsclient_test.go

@@ -30,9 +30,13 @@ const (
 	dockerContainerType = cns.Docker
 )
 
+var (
+	dnsservers = []string{"8.8.8.8", "8.8.4.4"}
+)
+
 func addTestStateToRestServer(t *testing.T, secondaryIps []string) {
 	var ipConfig cns.IPConfiguration
-	ipConfig.DNSServers = []string{"8.8.8.8", "8.8.4.4"}
+	ipConfig.DNSServers = dnsservers
 	ipConfig.GatewayIPAddress = gatewayIp
 	var ipSubnet cns.IPSubnet
 	ipSubnet.IPAddress = primaryIp
@@ -64,7 +68,7 @@ func addTestStateToRestServer(t *testing.T, secondaryIps []string) {
 	}
 }
 
-func getIPConfigFromGetNetworkContainerResponse(resp *cns.GetIPConfigResponse) (net.IPNet, error) {
+func getIPNetFromResponse(resp *cns.GetIPConfigResponse) (net.IPNet, error) {
 	var (
 		resultIPnet net.IPNet
 		err         error
@@ -180,7 +184,16 @@ func TestCNSClientRequestAndRelease(t *testing.T) {
 		t.Fatalf("get IP from CNS failed with %+v", err)
 	}
 
-	resultIPnet, err := getIPConfigFromGetNetworkContainerResponse(resp)
+	// validate gateway and dnsservers
+	if reflect.DeepEqual(resp.IPConfiguration.DNSServers, dnsservers) != true {
+		t.Fatalf("DnsServer is not added as expected ipConfig %+v, expected dnsServers: %+v", resp.IPConfiguration, dnsservers)
+	}
+
+	if reflect.DeepEqual(resp.IPConfiguration.GatewayIPAddress, gatewayIp) != true {
+		t.Fatalf("Gateway is not added as expected ipConfig %+v, expected GatewayIp: %+v", resp.IPConfiguration, gatewayIp)
+	}
+
+	resultIPnet, err := getIPNetFromResponse(resp)
 
 	if reflect.DeepEqual(desired, resultIPnet) != true {
 		t.Fatalf("Desired result not matching actual result, expected: %+v, actual: %+v", desired, resultIPnet)

cns/restserver/const.go

@@ -33,6 +33,8 @@ const (
 	InvalidSecondaryIPConfig                = 30
 	NetworkContainerPendingStatePropagation = 31
 	FailedToAllocateIpConfig                = 32
+	EmptyOrchestratorContext                = 33
+	UnsupportedOrchestratorContext          = 34
 	UnexpectedError                         = 99
 )
 

cns/restserver/internalapi_test.go

@@ -20,6 +20,10 @@ const (
 	dockerContainerType = cns.Docker
 )
 
+var (
+	dnsservers = []string{"8.8.8.8", "8.8.4.4"}
+)
+
 func TestCreateOrUpdateNetworkContainerInternal(t *testing.T) {
 	restartService()
 
@@ -181,11 +185,11 @@ func createAndValidateNCRequest(t *testing.T, secondaryIPConfigs map[string]cns.
 	if returnCode != 0 {
 		t.Fatalf("Failed to createNetworkContainerRequest, req: %+v, err: %d", req, returnCode)
 	}
-	validateNetworkRequest(t, req, false)
+	validateNetworkRequest(t, req)
 }
 
 // Validate the networkRequest is persisted.
-func validateNetworkRequest(t *testing.T, req cns.CreateNetworkContainerRequest, skipAvailableCheck bool) {
+func validateNetworkRequest(t *testing.T, req cns.CreateNetworkContainerRequest) {
 	containerStatus := svc.state.ContainerStatus[req.NetworkContainerid]
 
 	if containerStatus.ID != req.NetworkContainerid {
@@ -217,8 +221,21 @@ func validateNetworkRequest(t *testing.T, req cns.CreateNetworkContainerRequest,
 				}
 
 				// Validate IP state
-				if !skipAvailableCheck &&
-					ipStatus.State != cns.Available {
+				if ipStatus.OrchestratorContext != nil {
+					var podInfo cns.KubernetesPodInfo
+					if err := json.Unmarshal(ipStatus.OrchestratorContext, &podInfo); err != nil {
+						t.Fatalf("Failed to add IPConfig to state: %+v with error: %v", ipStatus, err)
+					}
+
+					if _, exists := svc.PodIPIDByOrchestratorContext[podInfo.GetOrchestratorContextKey()]; exists {
+						if ipStatus.State != cns.Allocated {
+							t.Fatalf("IPId: %s State is not Allocated, ipStatus: %+v", ipid, ipStatus)
+						}
+					} else {
+						t.Fatalf("Failed to find podContext for allocated ip: %+v, podinfo :%+v", ipStatus, podInfo)
+					}
+				} else if ipStatus.State != cns.Available {
+					// Todo: Validate for pendingRelease as well
 					t.Fatalf("IPId: %s State is not Available, ipStatus: %+v", ipid, ipStatus)
 				}
 
@@ -235,7 +252,7 @@ func validateNetworkRequest(t *testing.T, req cns.CreateNetworkContainerRequest,
 
 func generateNetworkContainerRequest(secondaryIps map[string]cns.SecondaryIPConfig, ncId string) cns.CreateNetworkContainerRequest {
 	var ipConfig cns.IPConfiguration
-	ipConfig.DNSServers = []string{"8.8.8.8", "8.8.4.4"}
+	ipConfig.DNSServers = dnsservers
 	ipConfig.GatewayIPAddress = gatewayIp
 	var ipSubnet cns.IPSubnet
 	ipSubnet.IPAddress = primaryIp
@@ -265,7 +282,7 @@ func validateNCStateAfterReconcile(t *testing.T, ncRequest *cns.CreateNetworkCon
 			t.Fatalf("CNS has some stale ContainerStatus, count: %d, state: %+v", len(svc.state.ContainerStatus), svc.state.ContainerStatus)
 		}
 	} else {
-		validateNetworkRequest(t, *ncRequest, true)
+		validateNetworkRequest(t, *ncRequest)
 	}
 
 	if len(expectedAllocatedPods) != len(svc.PodIPIDByOrchestratorContext) {

cns/restserver/ipam.go

@@ -4,6 +4,7 @@
 package restserver
 
 import (
+	"bytes"
 	"encoding/json"
 	"fmt"
 	"net/http"
@@ -17,7 +18,7 @@ func (service *HTTPRestService) requestIPConfigHandler(w http.ResponseWriter, r
 	var (
 		err             error
 		ipconfigRequest cns.GetIPConfigRequest
-		ipState         ipConfigurationStatus
+		ipconfiguration cns.IPConfiguration
 		returnCode      int
 		returnMessage   string
 	)
@@ -29,9 +30,12 @@ func (service *HTTPRestService) requestIPConfigHandler(w http.ResponseWriter, r
 	}
 
 	// retrieve ipconfig from nc
-	if ipState, err = requestIPConfigHelper(service, ipconfigRequest); err != nil {
-		returnCode = FailedToAllocateIpConfig
-		returnMessage = fmt.Sprintf("AllocateIPConfig failed: %v", err)
+	returnCode, returnMessage = service.validateIpConfigRequest(ipconfigRequest)
+	if returnCode == Success {
+		if ipconfiguration, err = requestIPConfigHelper(service, ipconfigRequest); err != nil {
+			returnCode = FailedToAllocateIpConfig
+			returnMessage = fmt.Sprintf("AllocateIPConfig failed: %v", err)
+		}
 	}
 
 	resp := cns.Response{
@@ -42,24 +46,26 @@ func (service *HTTPRestService) requestIPConfigHandler(w http.ResponseWriter, r
 	reserveResp := &cns.GetIPConfigResponse{
 		Response: resp,
 	}
-	reserveResp.IPConfiguration.IPSubnet = ipState.IPSubnet
+	reserveResp.IPConfiguration = ipconfiguration
 
 	err = service.Listener.Encode(w, &reserveResp)
 	logger.Response(service.Name, reserveResp, resp.ReturnCode, ReturnCodeToString(resp.ReturnCode), err)
 }
 
 func (service *HTTPRestService) releaseIPConfigHandler(w http.ResponseWriter, r *http.Request) {
 	var (
-		podInfo    cns.KubernetesPodInfo
-		req        cns.GetIPConfigRequest
-		statusCode int
+		podInfo       cns.KubernetesPodInfo
+		req           cns.GetIPConfigRequest
+		statusCode    int
+		returnMessage string
 	)
 
 	statusCode = UnexpectedError
 
 	err := service.Listener.Decode(w, r, &req)
 	logger.Request(service.Name, &req, err)
 	if err != nil {
+		returnMessage = err.Error()
 		return
 	}
 
@@ -68,25 +74,18 @@ func (service *HTTPRestService) releaseIPConfigHandler(w http.ResponseWriter, r
 
 		if err != nil {
 			resp.ReturnCode = statusCode
-			resp.Message = err.Error()
+			resp.Message = returnMessage
 		}
 
 		err = service.Listener.Encode(w, &resp)
 		logger.Response(service.Name, resp, resp.ReturnCode, ReturnCodeToString(resp.ReturnCode), err)
 	}()
 
-	if service.state.OrchestratorType != cns.KubernetesCRD {
-		err = fmt.Errorf("ReleaseIPConfig API supported only for kubernetes orchestrator")
-		return
-	}
-
-	// retrieve podinfo  from orchestrator context
-	if err = json.Unmarshal(req.OrchestratorContext, &podInfo); err != nil {
-		return
-	}
+	statusCode, returnMessage = service.validateIpConfigRequest(req)
 
-	if err = service.ReleaseIPConfig(podInfo); err != nil {
+	if err = service.releaseIPConfig(podInfo); err != nil {
 		statusCode = NotFound
+		returnMessage = err.Error()
 		return
 	}
 	return
@@ -139,7 +138,7 @@ func (service *HTTPRestService) setIPConfigAsAvailable(ipconfig ipConfigurationS
 ////SetIPConfigAsAllocated takes a lock of the service, and sets the ipconfig in the CNS stateas Available
 // Todo - CNI should also pass the IPAddress which needs to be released to validate if that is the right IP allcoated
 // in the first place.
-func (service *HTTPRestService) ReleaseIPConfig(podInfo cns.KubernetesPodInfo) error {
+func (service *HTTPRestService) releaseIPConfig(podInfo cns.KubernetesPodInfo) error {
 	service.Lock()
 	defer service.Unlock()
 
@@ -149,7 +148,7 @@ func (service *HTTPRestService) ReleaseIPConfig(podInfo cns.KubernetesPodInfo) e
 			service.setIPConfigAsAvailable(ipconfig, podInfo)
 		} else {
 			logger.Errorf("Failed to get release ipconfig. Pod to IPID exists, but IPID to IPConfig doesn't exist, CNS State potentially corrupt")
-			return fmt.Errorf("ReleaseIPConfig failed. Pod to IPID exists, but IPID to IPConfig doesn't exist, CNS State potentially corrupt")
+			return fmt.Errorf("releaseIPConfig failed. Pod to IPID exists, but IPID to IPConfig doesn't exist, CNS State potentially corrupt")
 		}
 	} else {
 		logger.Printf("SetIPConfigAsAvailable failed to release, no allocation found for pod")
@@ -158,81 +157,102 @@ func (service *HTTPRestService) ReleaseIPConfig(podInfo cns.KubernetesPodInfo) e
 	return nil
 }
 
-func (service *HTTPRestService) GetExistingIPConfig(podInfo cns.KubernetesPodInfo) (ipConfigurationStatus, bool, error) {
+func (service *HTTPRestService) GetExistingIPConfig(podInfo cns.KubernetesPodInfo) (cns.IPConfiguration, bool, error) {
 	var (
-		ipState ipConfigurationStatus
-		isExist bool
+		ipConfiguration cns.IPConfiguration
+		isExist         bool
 	)
 
 	service.RLock()
 	defer service.RUnlock()
 
 	ipID := service.PodIPIDByOrchestratorContext[podInfo.GetOrchestratorContextKey()]
 	if ipID != "" {
-		if ipState, isExist = service.PodIPConfigState[ipID]; isExist {
-			return ipState, isExist, nil
+		if ipState, isExist := service.PodIPConfigState[ipID]; isExist {
+			ipConfiguration.IPSubnet = ipState.IPSubnet
+			err := service.populateIpConfigInfoFromNCUntransacted(ipState.NCID, &ipConfiguration)
+			return ipConfiguration, isExist, err
 		}
+
 		logger.Errorf("Failed to get existing ipconfig. Pod to IPID exists, but IPID to IPConfig doesn't exist, CNS State potentially corrupt")
-		return ipState, isExist, fmt.Errorf("Failed to get existing ipconfig. Pod to IPID exists, but IPID to IPConfig doesn't exist, CNS State potentially corrupt")
+		return ipConfiguration, isExist, fmt.Errorf("Failed to get existing ipconfig. Pod to IPID exists, but IPID to IPConfig doesn't exist, CNS State potentially corrupt")
 	}
 
-	return ipState, isExist, nil
+	return ipConfiguration, isExist, nil
 }
 
-func (service *HTTPRestService) AllocateDesiredIPConfig(podInfo cns.KubernetesPodInfo, desiredIPAddress string, orchestratorContext json.RawMessage) (ipConfigurationStatus, error) {
-	var ipState ipConfigurationStatus
-
+func (service *HTTPRestService) AllocateDesiredIPConfig(podInfo cns.KubernetesPodInfo, desiredIPAddress string, orchestratorContext json.RawMessage) (cns.IPConfiguration, error) {
+	var ipConfiguration cns.IPConfiguration
 	service.Lock()
 	defer service.Unlock()
 
+	found := false
 	for _, ipState := range service.PodIPConfigState {
 		if ipState.IPSubnet.IPAddress == desiredIPAddress {
-			if ipState.State == cns.Available {
-				return service.setIPConfigAsAllocated(ipState, podInfo, orchestratorContext), nil
+			if ipState.State == cns.Allocated {
+				// This IP has already been allocated, if it is allocated to same pod, then return the same
+				// IPconfiguration
+				if bytes.Equal(orchestratorContext, ipState.OrchestratorContext) == true {
+					found = true
+				} else {
+					var pInfo cns.KubernetesPodInfo
+					json.Unmarshal(ipState.OrchestratorContext, &pInfo)
+					return ipConfiguration, fmt.Errorf("Desired IP is already allocated %+v to Pod: %+v, requested for pod %+v", ipState, pInfo, podInfo)
+				}
+			} else if ipState.State == cns.Available {
+				service.setIPConfigAsAllocated(ipState, podInfo, orchestratorContext)
+				found = true
+			} else {
+				return ipConfiguration, fmt.Errorf("Desired IP is not available %+v", ipState)
+			}
+
+			if found {
+				err := service.populateIpConfigInfoFromNCUntransacted(ipState.NCID, &ipConfiguration)
+				if err != nil {
+					return ipConfiguration, err
+				}
+
+				ipConfiguration.IPSubnet = ipState.IPSubnet
+				return ipConfiguration, nil
 			}
-			return ipState, fmt.Errorf("Desired IP has already been allocated")
 		}
 	}
-	return ipState, fmt.Errorf("Requested IP not found in pool")
+	return ipConfiguration, fmt.Errorf("Requested IP not found in pool")
 }
 
-func (service *HTTPRestService) AllocateAnyAvailableIPConfig(podInfo cns.KubernetesPodInfo, orchestratorContext json.RawMessage) (ipConfigurationStatus, error) {
-	var ipState ipConfigurationStatus
+func (service *HTTPRestService) AllocateAnyAvailableIPConfig(podInfo cns.KubernetesPodInfo, orchestratorContext json.RawMessage) (cns.IPConfiguration, error) {
+	var ipConfiguration cns.IPConfiguration
 
 	service.Lock()
 	defer service.Unlock()
 
-	for _, ipState = range service.PodIPConfigState {
+	for _, ipState := range service.PodIPConfigState {
 		if ipState.State == cns.Available {
-			return service.setIPConfigAsAllocated(ipState, podInfo, orchestratorContext), nil
+			service.setIPConfigAsAllocated(ipState, podInfo, orchestratorContext)
+
+			ipConfiguration.IPSubnet = ipState.IPSubnet
+			err := service.populateIpConfigInfoFromNCUntransacted(ipState.NCID, &ipConfiguration)
+			return ipConfiguration, err
 		}
 	}
-	return ipState, fmt.Errorf("No more free IP's available, trigger batch")
+
+	return ipConfiguration, fmt.Errorf("No more free IP's available, trigger batch")
 }
 
 // If IPConfig is already allocated for pod, it returns that else it returns one of the available ipconfigs.
-func requestIPConfigHelper(service *HTTPRestService, req cns.GetIPConfigRequest) (ipConfigurationStatus, error) {
+func requestIPConfigHelper(service *HTTPRestService, req cns.GetIPConfigRequest) (cns.IPConfiguration, error) {
 	var (
-		podInfo cns.KubernetesPodInfo
-		ipState ipConfigurationStatus
-		isExist bool
-		err     error
+		podInfo         cns.KubernetesPodInfo
+		ipConfiguration cns.IPConfiguration
+		isExist         bool
+		err             error
 	)
 
-	// todo - change it to
-	if service.state.OrchestratorType != cns.KubernetesCRD {
-		return ipState, fmt.Errorf("AllocateIPconfig API supported only for kubernetes orchestrator")
-	}
-
-	// retrieve podinfo  from orchestrator context
-	if err := json.Unmarshal(req.OrchestratorContext, &podInfo); err != nil {
-		return ipState, err
-	}
-
 	// check if ipconfig already allocated for this pod and return if exists or error
 	// if error, ipstate is nil, if exists, ipstate is not nil and error is nil
-	if ipState, isExist, err = service.GetExistingIPConfig(podInfo); err != nil || isExist {
-		return ipState, err
+	json.Unmarshal(req.OrchestratorContext, &podInfo)
+	if ipConfiguration, isExist, err = service.GetExistingIPConfig(podInfo); err != nil || isExist {
+		return ipConfiguration, err
 	}
 
 	// return desired IPConfig