From 9921109e028a2cf79c707dd8ed16ab101a8f056a Mon Sep 17 00:00:00 2001
From: vakr <vakr@microsoft.com>
Date: Mon, 28 Sep 2020 15:09:05 -0700
Subject: [PATCH 01/19] Adding greKey allocation support for mNAT and LB

---
 cns/api.go                         | 10 ++++++
 cns/nmagentclient/nmagentclient.go | 16 +++++++++
 cns/restserver/api.go              | 57 ++++++++++++++++++++++++++++++
 cns/restserver/const.go            |  1 +
 cns/restserver/restserver.go       |  2 ++
 5 files changed, 86 insertions(+)

diff --git a/cns/api.go b/cns/api.go
index 0c06500e28..9d35170893 100644
--- a/cns/api.go
+++ b/cns/api.go
@@ -27,6 +27,7 @@ const (
 	NumberOfCPUCoresPath          = "/hostcpucores"
 	CreateHostNCApipaEndpointPath = "/network/createhostncapipaendpoint"
 	DeleteHostNCApipaEndpointPath = "/network/deletehostncapipaendpoint"
+	NmAgentSupportedApisPath      = "/network/nmagentsupportedapis"
 	V1Prefix                      = "/v0.1"
 	V2Prefix                      = "/v0.2"
 )
@@ -212,3 +213,12 @@ type DeleteHostNCApipaEndpointRequest struct {
 type DeleteHostNCApipaEndpointResponse struct {
 	Response Response
 }
+
+type NmAgentSupportedApisRequest struct {
+	GetNmAgentSupportedApisURL string
+}
+
+type NmAgentSupportedApisResponse struct {
+	Response      Response
+	SupportedApis []string
+}
diff --git a/cns/nmagentclient/nmagentclient.go b/cns/nmagentclient/nmagentclient.go
index 4b0d2d7790..af6be620b7 100644
--- a/cns/nmagentclient/nmagentclient.go
+++ b/cns/nmagentclient/nmagentclient.go
@@ -20,6 +20,10 @@ type NMANetworkContainerResponse struct {
 	Version            string `json:"version"`
 }
 
+type NMAgentSupportedApisResponseXML struct {
+	SupportedApis []string `xml:"type"`
+}
+
 // JoinNetwork joins the given network
 func JoinNetwork(
 	networkID string,
@@ -86,3 +90,15 @@ func GetNetworkContainerVersion(
 		networkContainerID, response, err)
 	return response, err
 }
+
+// GetNmAgentSupportedApis :- Retrieves Supported Apis from NMAgent
+func GetNmAgentSupportedApis(
+	getNmAgentSupportedApisURL string) (*http.Response, error) {
+	logger.Printf("[NMAgentClient] In GetNmAgentSupportedApis func")
+
+	response, err := common.GetHttpClient().Get(getNmAgentSupportedApisURL)
+
+	logger.Printf("[NMAgentClient][Response] GetNmAgentSupportedApis. Response: %+v. Error: %v",
+		response, err)
+	return response, err
+}
diff --git a/cns/restserver/api.go b/cns/restserver/api.go
index 9a3ea73587..69a3479a32 100644
--- a/cns/restserver/api.go
+++ b/cns/restserver/api.go
@@ -4,6 +4,7 @@
 package restserver
 
 import (
+	"encoding/xml"
 	"fmt"
 	"io/ioutil"
 	"net"
@@ -1415,3 +1416,59 @@ func (service *HTTPRestService) deleteHostNCApipaEndpoint(w http.ResponseWriter,
 	err = service.Listener.Encode(w, &response)
 	logger.Response(service.Name, response, response.Response.ReturnCode, ReturnCodeToString(response.Response.ReturnCode), err)
 }
+
+// This function is used to query NMagents's supproted APIs list
+func (service *HTTPRestService) nmAgentSupportedApisHandler(w http.ResponseWriter, r *http.Request) {
+	logger.Printf("[Azure CNS] nmAgentSupportedApisHandler")
+	logger.Request(service.Name, "nmAgentSupportedApisHandler", nil)
+	var (
+		err           error
+		req           cns.NmAgentSupportedApisRequest
+		returnCode    int
+		returnMessage string
+		supportedApis []string
+	)
+
+	err = service.Listener.Decode(w, r, &req)
+	logger.Request(service.Name, &req, err)
+	if err != nil {
+		return
+	}
+
+	switch r.Method {
+	case "POST":
+		getApisResponse, getApisError := nmagentclient.GetNmAgentSupportedApis(
+			req.GetNmAgentSupportedApisURL)
+		if getApisError != nil || getApisResponse.StatusCode != http.StatusOK || getApisResponse == nil {
+			returnMessage = fmt.Sprintf("Failed to retrieve Supported Apis from NMAgent")
+			returnCode = NmAgentSupportedApisError
+			logger.Errorf("[Azure-CNS] %s", returnMessage)
+		}
+
+		if getApisResponse != nil {
+			var xmlDoc nmagentclient.NMAgentSupportedApisResponseXML
+			decoder := xml.NewDecoder(getApisResponse.Body)
+			err = decoder.Decode(&xmlDoc)
+			if err != nil {
+				returnMessage = fmt.Sprintf("Failed to decode XML response of Supported Apis from NMAgent")
+				returnCode = NmAgentSupportedApisError
+				logger.Errorf("[Azure-CNS] %s", returnMessage)
+			}
+			returnCode = 0
+			supportedApis = xmlDoc.SupportedApis
+		}
+
+	default:
+		returnMessage = "[Azure-CNS] GetHostLocalIP API expects a GET."
+	}
+
+	resp := cns.Response{ReturnCode: returnCode, Message: returnMessage}
+	nmAgentSupportedApisResponse := &cns.NmAgentSupportedApisResponse{
+		Response:      resp,
+		SupportedApis: supportedApis,
+	}
+
+	serviceErr := service.Listener.Encode(w, &nmAgentSupportedApisResponse)
+
+	logger.Response(service.Name, nmAgentSupportedApisResponse, resp.ReturnCode, ReturnCodeToString(resp.ReturnCode), serviceErr)
+}
diff --git a/cns/restserver/const.go b/cns/restserver/const.go
index 0beda46f19..6951d3b992 100644
--- a/cns/restserver/const.go
+++ b/cns/restserver/const.go
@@ -35,6 +35,7 @@ const (
 	FailedToAllocateIpConfig                = 32
 	EmptyOrchestratorContext                = 33
 	UnsupportedOrchestratorContext          = 34
+	NmAgentSupportedApisError               = 35
 	UnexpectedError                         = 99
 )
 
diff --git a/cns/restserver/restserver.go b/cns/restserver/restserver.go
index cce2494743..5dc26c8026 100644
--- a/cns/restserver/restserver.go
+++ b/cns/restserver/restserver.go
@@ -176,6 +176,7 @@ func (service *HTTPRestService) Start(config *common.ServiceConfig) error {
 	listener.AddHandler(cns.UnpublishNetworkContainer, service.unpublishNetworkContainer)
 	listener.AddHandler(cns.RequestIPConfig, service.requestIPConfigHandler)
 	listener.AddHandler(cns.ReleaseIPConfig, service.releaseIPConfigHandler)
+	listener.AddHandler(cns.NmAgentSupportedApisPath, service.nmAgentSupportedApisHandler)
 
 	// handlers for v0.2
 	listener.AddHandler(cns.V2Prefix+cns.SetEnvironmentPath, service.setEnvironment)
@@ -199,6 +200,7 @@ func (service *HTTPRestService) Start(config *common.ServiceConfig) error {
 	listener.AddHandler(cns.V2Prefix+cns.NumberOfCPUCoresPath, service.getNumberOfCPUCores)
 	listener.AddHandler(cns.V2Prefix+cns.CreateHostNCApipaEndpointPath, service.createHostNCApipaEndpoint)
 	listener.AddHandler(cns.V2Prefix+cns.DeleteHostNCApipaEndpointPath, service.deleteHostNCApipaEndpoint)
+	listener.AddHandler(cns.V2Prefix+cns.NmAgentSupportedApisPath, service.nmAgentSupportedApisHandler)
 
 	// Initialize HTTP client to be reused in CNS
 	connectionTimeout, _ := service.GetOption(acn.OptHttpConnectionTimeout).(int)

From 7ed7e9179e1d4bfb5feddbf968f88d2056831a79 Mon Sep 17 00:00:00 2001
From: vakr <vakr@microsoft.com>
Date: Tue, 29 Sep 2020 12:16:32 -0700
Subject: [PATCH 02/19] Covering pull model for DNC-CNS nmAgentSupportedApis
 list

---
 cns/NetworkContainerContract.go    |  7 ++++
 cns/nmagentclient/nmagentclient.go | 36 ++++++++++++++++-
 cns/restserver/api.go              | 19 +--------
 cns/restserver/internalapi.go      | 62 ++++++++++++++++++++++++++++++
 cns/service/main.go                | 45 +---------------------
 common/utils.go                    |  2 +-
 6 files changed, 107 insertions(+), 64 deletions(-)

diff --git a/cns/NetworkContainerContract.go b/cns/NetworkContainerContract.go
index bbb7e91508..81a77acb67 100644
--- a/cns/NetworkContainerContract.go
+++ b/cns/NetworkContainerContract.go
@@ -343,4 +343,11 @@ func (networkContainerRequestPolicy *NetworkContainerRequestPolicies) Validate()
 type NodeInfoResponse struct {
 	NetworkContainers  []CreateNetworkContainerRequest
 	GetNCVersionURLFmt string
+	NmAgentApisMissing bool
+}
+
+// NodeRegisterRequest - Struct to hold the node register request.
+type NodeRegisterRequest struct {
+	NumCPU               int
+	NmAgentSupportedApis []string
 }
diff --git a/cns/nmagentclient/nmagentclient.go b/cns/nmagentclient/nmagentclient.go
index af6be620b7..213dd32198 100644
--- a/cns/nmagentclient/nmagentclient.go
+++ b/cns/nmagentclient/nmagentclient.go
@@ -3,6 +3,8 @@ package nmagentclient
 import (
 	"bytes"
 	"encoding/json"
+	"encoding/xml"
+	"fmt"
 	"net/http"
 
 	"github.com/Azure/azure-container-networking/cns/logger"
@@ -10,7 +12,11 @@ import (
 )
 
 const (
+	//WireServerIP - wire server ip
 	WireserverIP = "168.63.129.16"
+
+	//GetNmAgentSupportedApiURLFmt Api endpoint to get supported Apis of NMAgent
+	GetNmAgentSupportedApiURLFmt = "http://%s/machine/plugins/?comp=nmagent&type=GetSupportedApis"
 )
 
 // NMANetworkContainerResponse - NMAgent response.
@@ -93,12 +99,38 @@ func GetNetworkContainerVersion(
 
 // GetNmAgentSupportedApis :- Retrieves Supported Apis from NMAgent
 func GetNmAgentSupportedApis(
-	getNmAgentSupportedApisURL string) (*http.Response, error) {
+	getNmAgentSupportedApisURL string) ([]string, string) {
 	logger.Printf("[NMAgentClient] In GetNmAgentSupportedApis func")
+	var (
+		returnMessage string
+		supportedApis []string
+	)
+
+	if getNmAgentSupportedApisURL == "" {
+		getNmAgentSupportedApisURL = fmt.Sprintf(
+			GetNmAgentSupportedApiURLFmt, WireserverIP)
+	}
 
 	response, err := common.GetHttpClient().Get(getNmAgentSupportedApisURL)
+	if err != nil || response.StatusCode != http.StatusOK || response == nil {
+		returnMessage = fmt.Sprintf("Failed to retrieve Supported Apis from NMAgent")
+		logger.Errorf("[Azure-CNS] %s", returnMessage)
+		return supportedApis, returnMessage
+	}
+
+	if response != nil {
+		var xmlDoc NMAgentSupportedApisResponseXML
+		decoder := xml.NewDecoder(response.Body)
+		err = decoder.Decode(&xmlDoc)
+		if err != nil {
+			returnMessage = fmt.Sprintf("Failed to decode XML response of Supported Apis from NMAgent")
+			logger.Errorf("[Azure-CNS] %s", returnMessage)
+			return supportedApis, returnMessage
+		}
+		supportedApis = xmlDoc.SupportedApis
+	}
 
 	logger.Printf("[NMAgentClient][Response] GetNmAgentSupportedApis. Response: %+v. Error: %v",
 		response, err)
-	return response, err
+	return supportedApis, ""
 }
diff --git a/cns/restserver/api.go b/cns/restserver/api.go
index 69a3479a32..2fd6f61162 100644
--- a/cns/restserver/api.go
+++ b/cns/restserver/api.go
@@ -4,7 +4,6 @@
 package restserver
 
 import (
-	"encoding/xml"
 	"fmt"
 	"io/ioutil"
 	"net"
@@ -1437,27 +1436,13 @@ func (service *HTTPRestService) nmAgentSupportedApisHandler(w http.ResponseWrite
 
 	switch r.Method {
 	case "POST":
-		getApisResponse, getApisError := nmagentclient.GetNmAgentSupportedApis(
+		supportedApis, returnMessage = nmagentclient.GetNmAgentSupportedApis(
 			req.GetNmAgentSupportedApisURL)
-		if getApisError != nil || getApisResponse.StatusCode != http.StatusOK || getApisResponse == nil {
-			returnMessage = fmt.Sprintf("Failed to retrieve Supported Apis from NMAgent")
+		if returnMessage != "" {
 			returnCode = NmAgentSupportedApisError
 			logger.Errorf("[Azure-CNS] %s", returnMessage)
 		}
 
-		if getApisResponse != nil {
-			var xmlDoc nmagentclient.NMAgentSupportedApisResponseXML
-			decoder := xml.NewDecoder(getApisResponse.Body)
-			err = decoder.Decode(&xmlDoc)
-			if err != nil {
-				returnMessage = fmt.Sprintf("Failed to decode XML response of Supported Apis from NMAgent")
-				returnCode = NmAgentSupportedApisError
-				logger.Errorf("[Azure-CNS] %s", returnMessage)
-			}
-			returnCode = 0
-			supportedApis = xmlDoc.SupportedApis
-		}
-
 	default:
 		returnMessage = "[Azure-CNS] GetHostLocalIP API expects a GET."
 	}
diff --git a/cns/restserver/internalapi.go b/cns/restserver/internalapi.go
index f82046cbfb..96b6af733e 100644
--- a/cns/restserver/internalapi.go
+++ b/cns/restserver/internalapi.go
@@ -10,6 +10,9 @@ import (
 	"net/http"
 	"net/http/httptest"
 	"reflect"
+	"runtime"
+	"strconv"
+	"time"
 
 	"github.com/Azure/azure-container-networking/cns"
 	"github.com/Azure/azure-container-networking/cns/logger"
@@ -132,6 +135,11 @@ func (service *HTTPRestService) SyncNodeStatus(dncEP, infraVnet, nodeID string,
 	service.saveState()
 	service.Unlock()
 
+	if nodeInfoResponse.NmAgentApisMissing {
+		// RegisterNode again with NmAgent Apis list
+		RegisterNode(service, dncEP, infraVnet, nodeID)
+	}
+
 	// delete dangling NCs
 	for nc := range ncsToBeDeleted {
 		var body bytes.Buffer
@@ -254,3 +262,57 @@ func (service *HTTPRestService) CreateOrUpdateNetworkContainerInternal(req cns.C
 	return returnCode
 
 }
+
+// Try to register node with DNC when CNS is started in managed DNC mode
+func RegisterNode(httpRestService cns.HTTPService, dncEP, infraVnet, nodeID string) {
+	logger.Printf("[Azure CNS] Registering node %s with Infrastructure Network: %s PrivateEndpoint: %s", nodeID, infraVnet, dncEP)
+
+	var (
+		numCPU              = runtime.NumCPU()
+		url                 = fmt.Sprintf(common.RegisterNodeURLFmt, dncEP, infraVnet, nodeID, dncApiVersion)
+		response            *http.Response
+		err                 = fmt.Errorf("")
+		body                bytes.Buffer
+		httpc               = common.GetHttpClient()
+		nodeRegisterRequest cns.NodeRegisterRequest
+	)
+
+	nodeRegisterRequest.NumCPU = numCPU
+	supportedApis, msg := nmagentclient.GetNmAgentSupportedApis("")
+
+	if msg != "" {
+		logger.Printf("[Azure CNS] Failed to retrieve SupportedApis from NMagent of node %s with Infrastructure Network: %s PrivateEndpoint: %s",
+			nodeID, infraVnet, dncEP)
+	}
+
+	nodeRegisterRequest.NmAgentSupportedApis = supportedApis
+	if err := json.NewEncoder(&body).Encode(nodeRegisterRequest); err != nil {
+		log.Errorf("encoding json failed with %v", err)
+		return
+	}
+
+	for sleep := true; err != nil; sleep = true {
+		response, err = httpc.Post(url, "application/json", &body)
+		if err == nil {
+			if response.StatusCode == http.StatusCreated {
+				var req cns.SetOrchestratorTypeRequest
+				json.NewDecoder(response.Body).Decode(&req)
+				httpRestService.SetNodeOrchestrator(&req)
+				sleep = false
+			} else {
+				err = fmt.Errorf("[Azure CNS] Failed to register node with http status code %s", strconv.Itoa(response.StatusCode))
+				logger.Errorf(err.Error())
+			}
+
+			response.Body.Close()
+		} else {
+			logger.Errorf("[Azure CNS] Failed to register node with err: %+v", err)
+		}
+
+		if sleep {
+			time.Sleep(common.FiveSeconds)
+		}
+	}
+
+	logger.Printf("[Azure CNS] Node Registered")
+}
diff --git a/cns/service/main.go b/cns/service/main.go
index e6b256479a..81d81cf3df 100644
--- a/cns/service/main.go
+++ b/cns/service/main.go
@@ -4,15 +4,11 @@
 package main
 
 import (
-	"bytes"
 	"context"
 	"encoding/json"
 	"fmt"
-	"net/http"
 	"os"
 	"os/signal"
-	"runtime"
-	"strconv"
 	"strings"
 	"syscall"
 	"time"
@@ -230,45 +226,6 @@ func printVersion() {
 	fmt.Printf("Version %v\n", version)
 }
 
-// Try to register node with DNC when CNS is started in managed DNC mode
-func registerNode(httpRestService cns.HTTPService, dncEP, infraVnet, nodeID string) {
-	logger.Printf("[Azure CNS] Registering node %s with Infrastructure Network: %s PrivateEndpoint: %s", nodeID, infraVnet, dncEP)
-
-	var (
-		numCPU   = runtime.NumCPU()
-		url      = fmt.Sprintf(acn.RegisterNodeURLFmt, dncEP, infraVnet, nodeID, numCPU, dncApiVersion)
-		response *http.Response
-		err      = fmt.Errorf("")
-		body     bytes.Buffer
-		httpc    = acn.GetHttpClient()
-	)
-
-	for sleep := true; err != nil; sleep = true {
-		response, err = httpc.Post(url, "application/json", &body)
-		if err == nil {
-			if response.StatusCode == http.StatusCreated {
-				var req cns.SetOrchestratorTypeRequest
-				json.NewDecoder(response.Body).Decode(&req)
-				httpRestService.SetNodeOrchestrator(&req)
-				sleep = false
-			} else {
-				err = fmt.Errorf("[Azure CNS] Failed to register node with http status code %s", strconv.Itoa(response.StatusCode))
-				logger.Errorf(err.Error())
-			}
-
-			response.Body.Close()
-		} else {
-			logger.Errorf("[Azure CNS] Failed to register node with err: %+v", err)
-		}
-
-		if sleep {
-			time.Sleep(acn.FiveSeconds)
-		}
-	}
-
-	logger.Printf("[Azure CNS] Node Registered")
-}
-
 // Main is the entry point for CNS.
 func main() {
 	// Initialize and parse command line arguments.
@@ -425,7 +382,7 @@ func main() {
 		httpRestService.SetOption(acn.OptInfrastructureNetworkID, infravnet)
 		httpRestService.SetOption(acn.OptNodeID, nodeID)
 
-		registerNode(httpRestService, privateEndpoint, infravnet, nodeID)
+		restserver.RegisterNode(httpRestService, privateEndpoint, infravnet, nodeID)
 		go func(ep, vnet, node string) {
 			// Periodically poll DNC for node updates
 			for {
diff --git a/common/utils.go b/common/utils.go
index a1e77df6d4..00e36fd128 100644
--- a/common/utils.go
+++ b/common/utils.go
@@ -26,7 +26,7 @@ const (
 	azCloudUrl                      = "http://169.254.169.254/metadata/instance/compute/azEnvironment?api-version=2018-10-01&format=text"
 	httpConnectionTimeout           = 7
 	headerTimeout                   = 7
-	RegisterNodeURLFmt              = "%s/%s/node/%s/cores/%d%s"
+	RegisterNodeURLFmt              = "%s/%s/node/%s%s"
 	SyncNodeNetworkContainersURLFmt = "%s/%s/node/%s%s"
 	FiveSeconds                     = 5 * time.Second
 	JsonContent                     = "application/json; charset=UTF-8"

From 3e804a2ad3697c7262fdb6a5c0ac86f2c4f14a2e Mon Sep 17 00:00:00 2001
From: vakr <vakr@microsoft.com>
Date: Tue, 29 Sep 2020 16:35:11 -0700
Subject: [PATCH 03/19] Addressing comments

---
 cns/nmagentclient/nmagentclient.go | 25 +++++++++++++------------
 cns/restserver/api.go              |  5 ++---
 2 files changed, 15 insertions(+), 15 deletions(-)

diff --git a/cns/nmagentclient/nmagentclient.go b/cns/nmagentclient/nmagentclient.go
index 213dd32198..0c491e80da 100644
--- a/cns/nmagentclient/nmagentclient.go
+++ b/cns/nmagentclient/nmagentclient.go
@@ -100,7 +100,6 @@ func GetNetworkContainerVersion(
 // GetNmAgentSupportedApis :- Retrieves Supported Apis from NMAgent
 func GetNmAgentSupportedApis(
 	getNmAgentSupportedApisURL string) ([]string, string) {
-	logger.Printf("[NMAgentClient] In GetNmAgentSupportedApis func")
 	var (
 		returnMessage string
 		supportedApis []string
@@ -113,22 +112,24 @@ func GetNmAgentSupportedApis(
 
 	response, err := common.GetHttpClient().Get(getNmAgentSupportedApisURL)
 	if err != nil || response.StatusCode != http.StatusOK || response == nil {
-		returnMessage = fmt.Sprintf("Failed to retrieve Supported Apis from NMAgent")
+		returnMessage = fmt.Sprintf(
+			"Failed to retrieve Supported Apis from NMAgent with error %v",
+			err.Error())
 		logger.Errorf("[Azure-CNS] %s", returnMessage)
 		return supportedApis, returnMessage
 	}
 
-	if response != nil {
-		var xmlDoc NMAgentSupportedApisResponseXML
-		decoder := xml.NewDecoder(response.Body)
-		err = decoder.Decode(&xmlDoc)
-		if err != nil {
-			returnMessage = fmt.Sprintf("Failed to decode XML response of Supported Apis from NMAgent")
-			logger.Errorf("[Azure-CNS] %s", returnMessage)
-			return supportedApis, returnMessage
-		}
-		supportedApis = xmlDoc.SupportedApis
+	var xmlDoc NMAgentSupportedApisResponseXML
+	decoder := xml.NewDecoder(response.Body)
+	err = decoder.Decode(&xmlDoc)
+	if err != nil {
+		returnMessage = fmt.Sprintf(
+			"Failed to decode XML response of Supported Apis from NMAgent with error %v",
+			err.Error())
+		logger.Errorf("[Azure-CNS] %s", returnMessage)
+		return supportedApis, returnMessage
 	}
+	supportedApis = xmlDoc.SupportedApis
 
 	logger.Printf("[NMAgentClient][Response] GetNmAgentSupportedApis. Response: %+v. Error: %v",
 		response, err)
diff --git a/cns/restserver/api.go b/cns/restserver/api.go
index 2fd6f61162..a5dee0c39b 100644
--- a/cns/restserver/api.go
+++ b/cns/restserver/api.go
@@ -1418,7 +1418,6 @@ func (service *HTTPRestService) deleteHostNCApipaEndpoint(w http.ResponseWriter,
 
 // This function is used to query NMagents's supproted APIs list
 func (service *HTTPRestService) nmAgentSupportedApisHandler(w http.ResponseWriter, r *http.Request) {
-	logger.Printf("[Azure CNS] nmAgentSupportedApisHandler")
 	logger.Request(service.Name, "nmAgentSupportedApisHandler", nil)
 	var (
 		err           error
@@ -1435,7 +1434,7 @@ func (service *HTTPRestService) nmAgentSupportedApisHandler(w http.ResponseWrite
 	}
 
 	switch r.Method {
-	case "POST":
+	case http.MethodPost:
 		supportedApis, returnMessage = nmagentclient.GetNmAgentSupportedApis(
 			req.GetNmAgentSupportedApisURL)
 		if returnMessage != "" {
@@ -1444,7 +1443,7 @@ func (service *HTTPRestService) nmAgentSupportedApisHandler(w http.ResponseWrite
 		}
 
 	default:
-		returnMessage = "[Azure-CNS] GetHostLocalIP API expects a GET."
+		returnMessage = "[Azure-CNS] GetHostLocalIP API expects a POST method."
 	}
 
 	resp := cns.Response{ReturnCode: returnCode, Message: returnMessage}

From 4a03264576afb55b95c4921e00b84a7f96b878a7 Mon Sep 17 00:00:00 2001
From: vakr <vakr@microsoft.com>
Date: Wed, 30 Sep 2020 12:28:13 -0700
Subject: [PATCH 04/19] Adding test cases for register Node

---
 cns/nmagentclient/nmagentclient.go |  3 +-
 cns/restserver/api.go              |  3 +-
 cns/restserver/internalapi.go      | 15 ++++---
 cns/restserver/internalapi_test.go | 69 +++++++++++++++++++++++++++++-
 cns/service/main.go                |  2 +-
 5 files changed, 82 insertions(+), 10 deletions(-)

diff --git a/cns/nmagentclient/nmagentclient.go b/cns/nmagentclient/nmagentclient.go
index 0c491e80da..a5f021d258 100644
--- a/cns/nmagentclient/nmagentclient.go
+++ b/cns/nmagentclient/nmagentclient.go
@@ -99,6 +99,7 @@ func GetNetworkContainerVersion(
 
 // GetNmAgentSupportedApis :- Retrieves Supported Apis from NMAgent
 func GetNmAgentSupportedApis(
+	httpc *http.Client,
 	getNmAgentSupportedApisURL string) ([]string, string) {
 	var (
 		returnMessage string
@@ -110,7 +111,7 @@ func GetNmAgentSupportedApis(
 			GetNmAgentSupportedApiURLFmt, WireserverIP)
 	}
 
-	response, err := common.GetHttpClient().Get(getNmAgentSupportedApisURL)
+	response, err := httpc.Get(getNmAgentSupportedApisURL)
 	if err != nil || response.StatusCode != http.StatusOK || response == nil {
 		returnMessage = fmt.Sprintf(
 			"Failed to retrieve Supported Apis from NMAgent with error %v",
diff --git a/cns/restserver/api.go b/cns/restserver/api.go
index a5dee0c39b..83c76b0de7 100644
--- a/cns/restserver/api.go
+++ b/cns/restserver/api.go
@@ -15,6 +15,7 @@ import (
 	"github.com/Azure/azure-container-networking/cns/hnsclient"
 	"github.com/Azure/azure-container-networking/cns/logger"
 	"github.com/Azure/azure-container-networking/cns/nmagentclient"
+	"github.com/Azure/azure-container-networking/common"
 	"github.com/Azure/azure-container-networking/platform"
 )
 
@@ -1435,7 +1436,7 @@ func (service *HTTPRestService) nmAgentSupportedApisHandler(w http.ResponseWrite
 
 	switch r.Method {
 	case http.MethodPost:
-		supportedApis, returnMessage = nmagentclient.GetNmAgentSupportedApis(
+		supportedApis, returnMessage = nmagentclient.GetNmAgentSupportedApis(common.GetHttpClient(),
 			req.GetNmAgentSupportedApisURL)
 		if returnMessage != "" {
 			returnCode = NmAgentSupportedApisError
diff --git a/cns/restserver/internalapi.go b/cns/restserver/internalapi.go
index 96b6af733e..cae875c7c8 100644
--- a/cns/restserver/internalapi.go
+++ b/cns/restserver/internalapi.go
@@ -137,7 +137,7 @@ func (service *HTTPRestService) SyncNodeStatus(dncEP, infraVnet, nodeID string,
 
 	if nodeInfoResponse.NmAgentApisMissing {
 		// RegisterNode again with NmAgent Apis list
-		RegisterNode(service, dncEP, infraVnet, nodeID)
+		RegisterNode(httpc, service, dncEP, infraVnet, nodeID)
 	}
 
 	// delete dangling NCs
@@ -264,7 +264,7 @@ func (service *HTTPRestService) CreateOrUpdateNetworkContainerInternal(req cns.C
 }
 
 // Try to register node with DNC when CNS is started in managed DNC mode
-func RegisterNode(httpRestService cns.HTTPService, dncEP, infraVnet, nodeID string) {
+func RegisterNode(httpc *http.Client, httpRestService cns.HTTPService, dncEP, infraVnet, nodeID string) string {
 	logger.Printf("[Azure CNS] Registering node %s with Infrastructure Network: %s PrivateEndpoint: %s", nodeID, infraVnet, dncEP)
 
 	var (
@@ -273,12 +273,12 @@ func RegisterNode(httpRestService cns.HTTPService, dncEP, infraVnet, nodeID stri
 		response            *http.Response
 		err                 = fmt.Errorf("")
 		body                bytes.Buffer
-		httpc               = common.GetHttpClient()
 		nodeRegisterRequest cns.NodeRegisterRequest
+		retMsg              string
 	)
 
 	nodeRegisterRequest.NumCPU = numCPU
-	supportedApis, msg := nmagentclient.GetNmAgentSupportedApis("")
+	supportedApis, msg := nmagentclient.GetNmAgentSupportedApis(httpc, "")
 
 	if msg != "" {
 		logger.Printf("[Azure CNS] Failed to retrieve SupportedApis from NMagent of node %s with Infrastructure Network: %s PrivateEndpoint: %s",
@@ -287,8 +287,9 @@ func RegisterNode(httpRestService cns.HTTPService, dncEP, infraVnet, nodeID stri
 
 	nodeRegisterRequest.NmAgentSupportedApis = supportedApis
 	if err := json.NewEncoder(&body).Encode(nodeRegisterRequest); err != nil {
-		log.Errorf("encoding json failed with %v", err)
-		return
+		retMsg = fmt.Sprintf("encoding json failed with %v", err)
+		log.Errorf(retMsg)
+		return retMsg
 	}
 
 	for sleep := true; err != nil; sleep = true {
@@ -302,6 +303,7 @@ func RegisterNode(httpRestService cns.HTTPService, dncEP, infraVnet, nodeID stri
 			} else {
 				err = fmt.Errorf("[Azure CNS] Failed to register node with http status code %s", strconv.Itoa(response.StatusCode))
 				logger.Errorf(err.Error())
+				return err.Error()
 			}
 
 			response.Body.Close()
@@ -315,4 +317,5 @@ func RegisterNode(httpRestService cns.HTTPService, dncEP, infraVnet, nodeID stri
 	}
 
 	logger.Printf("[Azure CNS] Node Registered")
+	return retMsg
 }
diff --git a/cns/restserver/internalapi_test.go b/cns/restserver/internalapi_test.go
index b6721bf8d9..f32ea53202 100644
--- a/cns/restserver/internalapi_test.go
+++ b/cns/restserver/internalapi_test.go
@@ -4,10 +4,15 @@
 package restserver
 
 import (
+	"bytes"
 	"encoding/json"
 	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
 	"reflect"
 	"strconv"
+	"strings"
 	"testing"
 
 	"github.com/Azure/azure-container-networking/cns"
@@ -27,7 +32,16 @@ const (
 )
 
 var (
-	dnsservers = []string{"8.8.8.8", "8.8.4.4"}
+	dnsservers        = []string{"8.8.8.8", "8.8.4.4"}
+	hostSupportedApis = `<SupportedRequestTypes>
+							<type>GetSupportedApis</type>
+							<type>GetIpRangesV1</type>
+							<type>GetIpRangesV2</type>
+							<type>GetInterfaceInfoV1</type>
+							<type>PortContainerIOVInformationV1</type>
+							<type>NetworkManagement</type>
+							<type>NetworkManagementDNSSupport</type>
+						</SupportedRequestTypes>`
 )
 
 func TestCreateOrUpdateNetworkContainerInternal(t *testing.T) {
@@ -130,6 +144,17 @@ func TestReconcileNCWithSystemPods(t *testing.T) {
 	validateNCStateAfterReconcile(t, &req, expectedNcCount, expectedAllocatedPods)
 }
 
+func TestRegisterNode(t *testing.T) {
+	restartService()
+	setEnv(t)
+	setOrchestratorTypeInternal(cns.KubernetesCRD)
+
+	err := RegisterNode(NewTestClient(), svc, "localhost", "dummyvnet", "dummyNodeId")
+	if err != "" {
+		t.Errorf("Unexpected failure on register Node %s", err)
+	}
+}
+
 func setOrchestratorTypeInternal(orchestratorType string) {
 	fmt.Println("setOrchestratorTypeInternal")
 	svc.state.OrchestratorType = orchestratorType
@@ -347,3 +372,45 @@ func restartService() {
 	service.Stop()
 	startService()
 }
+
+// RoundTripFunc .
+type RoundTripFunc func(req *http.Request) *http.Response
+
+// RoundTrip .
+func (f RoundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) {
+	return f(req), nil
+}
+
+func mockRountTrip(req *http.Request) *http.Response {
+	var (
+		body       io.ReadCloser
+		returnCode = 200
+	)
+	// Test request parameters
+	//equals(t, req.URL.String(), "http://example.com/some/path")
+	if strings.Contains(req.URL.String(), "GetSupportedApis") {
+		// Handle Call to NMAgent
+		body = ioutil.NopCloser(bytes.NewBufferString(hostSupportedApis))
+
+	} else if strings.Contains(req.URL.String(), "dummyNodeId") {
+		//Handle Call to register Node
+		body = ioutil.NopCloser(bytes.NewBufferString("OK"))
+		returnCode = 201
+
+	}
+
+	return &http.Response{
+		StatusCode: returnCode,
+		// Send response to be tested
+		Body: body,
+		// Must be set to non-nil value or it panics
+		Header: make(http.Header),
+	}
+}
+
+//NewTestClient returns *http.Client with Transport replaced to avoid making real calls
+func NewTestClient() *http.Client {
+	return &http.Client{
+		Transport: RoundTripFunc(mockRountTrip),
+	}
+}
diff --git a/cns/service/main.go b/cns/service/main.go
index 81d81cf3df..7cda9e224a 100644
--- a/cns/service/main.go
+++ b/cns/service/main.go
@@ -382,7 +382,7 @@ func main() {
 		httpRestService.SetOption(acn.OptInfrastructureNetworkID, infravnet)
 		httpRestService.SetOption(acn.OptNodeID, nodeID)
 
-		restserver.RegisterNode(httpRestService, privateEndpoint, infravnet, nodeID)
+		restserver.RegisterNode(acn.GetHttpClient(), httpRestService, privateEndpoint, infravnet, nodeID)
 		go func(ep, vnet, node string) {
 			// Periodically poll DNC for node updates
 			for {

From bc2b7285123b07232829196c58ef2327573e5669 Mon Sep 17 00:00:00 2001
From: vakr <vakr@microsoft.com>
Date: Wed, 30 Sep 2020 14:41:07 -0700
Subject: [PATCH 05/19] Adding one more testcase

---
 cns/restserver/api_test.go | 32 ++++++++++++++++++++++++++++++++
 1 file changed, 32 insertions(+)

diff --git a/cns/restserver/api_test.go b/cns/restserver/api_test.go
index 3cc1cc0559..4be9342d92 100644
--- a/cns/restserver/api_test.go
+++ b/cns/restserver/api_test.go
@@ -416,6 +416,38 @@ func TestUnpublishNCViaCNS(t *testing.T) {
 	fmt.Printf("UnpublishNetworkContainer succeded with response %+v, raw:%+v\n", resp, w.Body)
 }
 
+func TestNmAgentSupportedApisHandler(t *testing.T) {
+	fmt.Println("Test: nmAgentSupportedApisHandler")
+
+	var (
+		err        error
+		req        *http.Request
+		nmAgentReq cns.NmAgentSupportedApisRequest
+		body       bytes.Buffer
+	)
+
+	json.NewEncoder(&body).Encode(nmAgentReq)
+	req, err = http.NewRequest(http.MethodGet, cns.NmAgentSupportedApisPath, &body)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	var w *httptest.ResponseRecorder
+	w = httptest.NewRecorder()
+	mux.ServeHTTP(w, req)
+	var nmAgentSupportedApisResponse cns.NmAgentSupportedApisResponse
+
+	err = decodeResponse(w, &nmAgentSupportedApisResponse)
+	if err != nil || nmAgentSupportedApisResponse.Response.ReturnCode != 0 {
+		t.Errorf("nmAgentSupportedApisHandler failed with response %+v", nmAgentSupportedApisResponse)
+	}
+
+	// Since we are testing the NMAgent API in internalapi_test, we will skip POST call
+	// and test other paths
+	fmt.Printf("nmAgentSupportedApisHandler Responded with %+v\n", nmAgentSupportedApisResponse)
+
+}
+
 func setOrchestratorType(t *testing.T, orchestratorType string) error {
 	var body bytes.Buffer
 

From e9e0686608e130d108c2942878c5faaca1b04f5d Mon Sep 17 00:00:00 2001
From: vakr <vakr@microsoft.com>
Date: Wed, 30 Sep 2020 15:38:59 -0700
Subject: [PATCH 06/19] Adding in export function to checkk all test env
 variables

---
 .pipelines/e2e-step-template.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.pipelines/e2e-step-template.yaml b/.pipelines/e2e-step-template.yaml
index e5a351c296..4f3479c3f6 100644
--- a/.pipelines/e2e-step-template.yaml
+++ b/.pipelines/e2e-step-template.yaml
@@ -89,6 +89,7 @@ steps:
       export REGIONS=$(AKS_ENGINE_REGION) 
       export IS_JENKINS=false 
       export DEBUG_CRASHING_PODS=true
+      export
       make test-kubernetes
     name: DeployAKSEngine
     displayName: Run AKS-Engine E2E Tests

From 42cf7d1a1ea88564e7e331a3bf9ad7cdd708c912 Mon Sep 17 00:00:00 2001
From: vakr <vakr@microsoft.com>
Date: Wed, 30 Sep 2020 22:39:06 -0700
Subject: [PATCH 07/19] addressing comments and adding one test

---
 cns/nmagentclient/nmagentclient.go | 31 ++++++++++++++++--------------
 cns/restserver/api.go              | 10 +++++-----
 cns/restserver/api_test.go         | 30 +++++++++++++++++++++++++++++
 cns/restserver/internalapi.go      |  4 ++--
 cns/restserver/internalapi_test.go |  8 +++++---
 5 files changed, 59 insertions(+), 24 deletions(-)

diff --git a/cns/nmagentclient/nmagentclient.go b/cns/nmagentclient/nmagentclient.go
index a5f021d258..6e02d64df6 100644
--- a/cns/nmagentclient/nmagentclient.go
+++ b/cns/nmagentclient/nmagentclient.go
@@ -100,10 +100,9 @@ func GetNetworkContainerVersion(
 // GetNmAgentSupportedApis :- Retrieves Supported Apis from NMAgent
 func GetNmAgentSupportedApis(
 	httpc *http.Client,
-	getNmAgentSupportedApisURL string) ([]string, string) {
+	getNmAgentSupportedApisURL string) ([]string, error) {
 	var (
-		returnMessage string
-		supportedApis []string
+		returnErr error
 	)
 
 	if getNmAgentSupportedApisURL == "" {
@@ -112,27 +111,31 @@ func GetNmAgentSupportedApis(
 	}
 
 	response, err := httpc.Get(getNmAgentSupportedApisURL)
-	if err != nil || response.StatusCode != http.StatusOK || response == nil {
-		returnMessage = fmt.Sprintf(
-			"Failed to retrieve Supported Apis from NMAgent with error %v",
-			err.Error())
-		logger.Errorf("[Azure-CNS] %s", returnMessage)
-		return supportedApis, returnMessage
+	if response.StatusCode != http.StatusOK || response == nil {
+		returnErr = fmt.Errorf(
+			"Failed to retrieve Supported Apis from NMAgent with StatusCode: %d",
+			response.StatusCode)
+		if err != nil {
+			returnErr = fmt.Errorf(
+				"Failed to retrieve Supported Apis from NMAgent with error %v",
+				err.Error())
+		}
+		logger.Errorf("[Azure-CNS] %s", returnErr)
+		return []string{}, returnErr
 	}
 
 	var xmlDoc NMAgentSupportedApisResponseXML
 	decoder := xml.NewDecoder(response.Body)
 	err = decoder.Decode(&xmlDoc)
 	if err != nil {
-		returnMessage = fmt.Sprintf(
+		returnErr = fmt.Errorf(
 			"Failed to decode XML response of Supported Apis from NMAgent with error %v",
 			err.Error())
-		logger.Errorf("[Azure-CNS] %s", returnMessage)
-		return supportedApis, returnMessage
+		logger.Errorf("[Azure-CNS] %s", returnErr)
+		return []string{}, returnErr
 	}
-	supportedApis = xmlDoc.SupportedApis
 
 	logger.Printf("[NMAgentClient][Response] GetNmAgentSupportedApis. Response: %+v. Error: %v",
 		response, err)
-	return supportedApis, ""
+	return xmlDoc.SupportedApis, nil
 }
diff --git a/cns/restserver/api.go b/cns/restserver/api.go
index 83c76b0de7..35060b5392 100644
--- a/cns/restserver/api.go
+++ b/cns/restserver/api.go
@@ -1421,7 +1421,7 @@ func (service *HTTPRestService) deleteHostNCApipaEndpoint(w http.ResponseWriter,
 func (service *HTTPRestService) nmAgentSupportedApisHandler(w http.ResponseWriter, r *http.Request) {
 	logger.Request(service.Name, "nmAgentSupportedApisHandler", nil)
 	var (
-		err           error
+		err, retErr   error
 		req           cns.NmAgentSupportedApisRequest
 		returnCode    int
 		returnMessage string
@@ -1436,15 +1436,15 @@ func (service *HTTPRestService) nmAgentSupportedApisHandler(w http.ResponseWrite
 
 	switch r.Method {
 	case http.MethodPost:
-		supportedApis, returnMessage = nmagentclient.GetNmAgentSupportedApis(common.GetHttpClient(),
+		supportedApis, retErr = nmagentclient.GetNmAgentSupportedApis(common.GetHttpClient(),
 			req.GetNmAgentSupportedApisURL)
-		if returnMessage != "" {
+		if retErr != nil {
 			returnCode = NmAgentSupportedApisError
-			logger.Errorf("[Azure-CNS] %s", returnMessage)
+			returnMessage = fmt.Sprintf("[Azure-CNS] %s", retErr.Error())
 		}
 
 	default:
-		returnMessage = "[Azure-CNS] GetHostLocalIP API expects a POST method."
+		returnMessage = "[Azure-CNS] NmAgentSupported API list expects a POST method."
 	}
 
 	resp := cns.Response{ReturnCode: returnCode, Message: returnMessage}
diff --git a/cns/restserver/api_test.go b/cns/restserver/api_test.go
index 4be9342d92..32ca7d5f9d 100644
--- a/cns/restserver/api_test.go
+++ b/cns/restserver/api_test.go
@@ -448,6 +448,36 @@ func TestNmAgentSupportedApisHandler(t *testing.T) {
 
 }
 
+func TestCreateHostNCApipaEndpoint(t *testing.T) {
+	fmt.Println("Test: createHostNCApipaEndpoint")
+
+	var (
+		err           error
+		req           *http.Request
+		createHostReq cns.CreateHostNCApipaEndpointRequest
+		body          bytes.Buffer
+	)
+
+	json.NewEncoder(&body).Encode(createHostReq)
+	req, err = http.NewRequest(http.MethodPost, cns.CreateHostNCApipaEndpointPath, &body)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	var w *httptest.ResponseRecorder
+	w = httptest.NewRecorder()
+	mux.ServeHTTP(w, req)
+	var createHostNCApipaEndpointResponse cns.CreateHostNCApipaEndpointResponse
+
+	err = decodeResponse(w, &createHostNCApipaEndpointResponse)
+	if err != nil || createHostNCApipaEndpointResponse.Response.ReturnCode != 0 {
+		t.Errorf("createHostNCApipaEndpoint failed with response %+v", createHostNCApipaEndpointResponse)
+	}
+
+	fmt.Printf("createHostNCApipaEndpoint Responded with %+v\n", createHostNCApipaEndpointResponse)
+
+}
+
 func setOrchestratorType(t *testing.T, orchestratorType string) error {
 	var body bytes.Buffer
 
diff --git a/cns/restserver/internalapi.go b/cns/restserver/internalapi.go
index cae875c7c8..136476e4e9 100644
--- a/cns/restserver/internalapi.go
+++ b/cns/restserver/internalapi.go
@@ -278,9 +278,9 @@ func RegisterNode(httpc *http.Client, httpRestService cns.HTTPService, dncEP, in
 	)
 
 	nodeRegisterRequest.NumCPU = numCPU
-	supportedApis, msg := nmagentclient.GetNmAgentSupportedApis(httpc, "")
+	supportedApis, retErr := nmagentclient.GetNmAgentSupportedApis(httpc, "")
 
-	if msg != "" {
+	if retErr != nil {
 		logger.Printf("[Azure CNS] Failed to retrieve SupportedApis from NMagent of node %s with Infrastructure Network: %s PrivateEndpoint: %s",
 			nodeID, infraVnet, dncEP)
 	}
diff --git a/cns/restserver/internalapi_test.go b/cns/restserver/internalapi_test.go
index f32ea53202..ab25df3ef1 100644
--- a/cns/restserver/internalapi_test.go
+++ b/cns/restserver/internalapi_test.go
@@ -387,16 +387,18 @@ func mockRountTrip(req *http.Request) *http.Response {
 		returnCode = 200
 	)
 	// Test request parameters
-	//equals(t, req.URL.String(), "http://example.com/some/path")
-	if strings.Contains(req.URL.String(), "GetSupportedApis") {
+	switch {
+	case strings.Contains(req.URL.String(), "GetSupportedApis"):
 		// Handle Call to NMAgent
 		body = ioutil.NopCloser(bytes.NewBufferString(hostSupportedApis))
 
-	} else if strings.Contains(req.URL.String(), "dummyNodeId") {
+	case strings.Contains(req.URL.String(), "dummyNodeId"):
 		//Handle Call to register Node
 		body = ioutil.NopCloser(bytes.NewBufferString("OK"))
 		returnCode = 201
 
+	default:
+		returnCode = 200
 	}
 
 	return &http.Response{

From 0084f532cb63e427af278c52c3efd29749322dc3 Mon Sep 17 00:00:00 2001
From: vakr <vakr@microsoft.com>
Date: Wed, 30 Sep 2020 23:08:48 -0700
Subject: [PATCH 08/19] changing fail criteria for a test

---
 cns/restserver/api_test.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cns/restserver/api_test.go b/cns/restserver/api_test.go
index 32ca7d5f9d..68ddf58efc 100644
--- a/cns/restserver/api_test.go
+++ b/cns/restserver/api_test.go
@@ -470,7 +470,7 @@ func TestCreateHostNCApipaEndpoint(t *testing.T) {
 	var createHostNCApipaEndpointResponse cns.CreateHostNCApipaEndpointResponse
 
 	err = decodeResponse(w, &createHostNCApipaEndpointResponse)
-	if err != nil || createHostNCApipaEndpointResponse.Response.ReturnCode != 0 {
+	if err != nil || createHostNCApipaEndpointResponse.Response.ReturnCode != UnknownContainerID {
 		t.Errorf("createHostNCApipaEndpoint failed with response %+v", createHostNCApipaEndpointResponse)
 	}
 

From 6b2acb480ca094b22cb2b43162f3e0f04074c405 Mon Sep 17 00:00:00 2001
From: vakr <vakr@microsoft.com>
Date: Thu, 1 Oct 2020 22:42:03 -0700
Subject: [PATCH 09/19] Addressing more comments

---
 cns/restserver/internalapi.go | 17 +++++++++--------
 cns/service/main.go           | 10 +++++++++-
 2 files changed, 18 insertions(+), 9 deletions(-)

diff --git a/cns/restserver/internalapi.go b/cns/restserver/internalapi.go
index 136476e4e9..3ca07137bf 100644
--- a/cns/restserver/internalapi.go
+++ b/cns/restserver/internalapi.go
@@ -137,7 +137,10 @@ func (service *HTTPRestService) SyncNodeStatus(dncEP, infraVnet, nodeID string,
 
 	if nodeInfoResponse.NmAgentApisMissing {
 		// RegisterNode again with NmAgent Apis list
-		RegisterNode(httpc, service, dncEP, infraVnet, nodeID)
+		retErr := RegisterNode(httpc, service, dncEP, infraVnet, nodeID)
+		if retErr != nil {
+			logger.Errorf("[Azure-CNS] Failed to register Node ID: %s with error: %s", nodeID, err.Error())
+		}
 	}
 
 	// delete dangling NCs
@@ -264,7 +267,7 @@ func (service *HTTPRestService) CreateOrUpdateNetworkContainerInternal(req cns.C
 }
 
 // Try to register node with DNC when CNS is started in managed DNC mode
-func RegisterNode(httpc *http.Client, httpRestService cns.HTTPService, dncEP, infraVnet, nodeID string) string {
+func RegisterNode(httpc *http.Client, httpRestService cns.HTTPService, dncEP, infraVnet, nodeID string) error {
 	logger.Printf("[Azure CNS] Registering node %s with Infrastructure Network: %s PrivateEndpoint: %s", nodeID, infraVnet, dncEP)
 
 	var (
@@ -274,7 +277,6 @@ func RegisterNode(httpc *http.Client, httpRestService cns.HTTPService, dncEP, in
 		err                 = fmt.Errorf("")
 		body                bytes.Buffer
 		nodeRegisterRequest cns.NodeRegisterRequest
-		retMsg              string
 	)
 
 	nodeRegisterRequest.NumCPU = numCPU
@@ -287,9 +289,8 @@ func RegisterNode(httpc *http.Client, httpRestService cns.HTTPService, dncEP, in
 
 	nodeRegisterRequest.NmAgentSupportedApis = supportedApis
 	if err := json.NewEncoder(&body).Encode(nodeRegisterRequest); err != nil {
-		retMsg = fmt.Sprintf("encoding json failed with %v", err)
-		log.Errorf(retMsg)
-		return retMsg
+		log.Errorf("encoding json failed with %v", err)
+		return err
 	}
 
 	for sleep := true; err != nil; sleep = true {
@@ -303,7 +304,7 @@ func RegisterNode(httpc *http.Client, httpRestService cns.HTTPService, dncEP, in
 			} else {
 				err = fmt.Errorf("[Azure CNS] Failed to register node with http status code %s", strconv.Itoa(response.StatusCode))
 				logger.Errorf(err.Error())
-				return err.Error()
+				return err
 			}
 
 			response.Body.Close()
@@ -317,5 +318,5 @@ func RegisterNode(httpc *http.Client, httpRestService cns.HTTPService, dncEP, in
 	}
 
 	logger.Printf("[Azure CNS] Node Registered")
-	return retMsg
+	return nil
 }
diff --git a/cns/service/main.go b/cns/service/main.go
index 7cda9e224a..436659f217 100644
--- a/cns/service/main.go
+++ b/cns/service/main.go
@@ -382,7 +382,15 @@ func main() {
 		httpRestService.SetOption(acn.OptInfrastructureNetworkID, infravnet)
 		httpRestService.SetOption(acn.OptNodeID, nodeID)
 
-		restserver.RegisterNode(acn.GetHttpClient(), httpRestService, privateEndpoint, infravnet, nodeID)
+		registerErr := restserver.RegisterNode(acn.GetHttpClient(), httpRestService, privateEndpoint, infravnet, nodeID)
+		if registerErr != nil {
+			logger.Errorf("[Azure CNS] Resgistering Node failed with error: %v PrivateEndpoint: %s InfrastructureNetworkID: %s NodeID: %s",
+				registerErr,
+				privateEndpoint,
+				infravnet,
+				nodeID)
+			return
+		}
 		go func(ep, vnet, node string) {
 			// Periodically poll DNC for node updates
 			for {

From 1220cda01da7c0a3e9a2b03eb2032674ad3423b0 Mon Sep 17 00:00:00 2001
From: vakr <vakr@microsoft.com>
Date: Fri, 2 Oct 2020 09:22:53 -0700
Subject: [PATCH 10/19] fixing a small typo

---
 cns/restserver/internalapi_test.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cns/restserver/internalapi_test.go b/cns/restserver/internalapi_test.go
index ab25df3ef1..723e7222d7 100644
--- a/cns/restserver/internalapi_test.go
+++ b/cns/restserver/internalapi_test.go
@@ -150,7 +150,7 @@ func TestRegisterNode(t *testing.T) {
 	setOrchestratorTypeInternal(cns.KubernetesCRD)
 
 	err := RegisterNode(NewTestClient(), svc, "localhost", "dummyvnet", "dummyNodeId")
-	if err != "" {
+	if err != nil {
 		t.Errorf("Unexpected failure on register Node %s", err)
 	}
 }

From 5dea9d9c11dd10e4dd8f3ca8b2f105cbdb7ebdb1 Mon Sep 17 00:00:00 2001
From: vakr <vakr@microsoft.com>
Date: Wed, 11 Nov 2020 13:51:06 -0800
Subject: [PATCH 11/19] Adding a ticker logic and cleaning up some comments

---
 cns/nmagentclient/nmagentclient.go | 26 +++++----
 cns/restserver/api.go              |  5 +-
 cns/restserver/internalapi.go      | 85 ++++++++++++++++++++----------
 3 files changed, 79 insertions(+), 37 deletions(-)

diff --git a/cns/nmagentclient/nmagentclient.go b/cns/nmagentclient/nmagentclient.go
index 6e02d64df6..05e53651fe 100644
--- a/cns/nmagentclient/nmagentclient.go
+++ b/cns/nmagentclient/nmagentclient.go
@@ -111,17 +111,25 @@ func GetNmAgentSupportedApis(
 	}
 
 	response, err := httpc.Get(getNmAgentSupportedApisURL)
-	if response.StatusCode != http.StatusOK || response == nil {
+	if err != nil {
+		returnErr = fmt.Errorf(
+			"Failed to retrieve Supported Apis from NMAgent with error %v",
+			err.Error())
+		logger.Errorf("[Azure-CNS] %s", returnErr)
+		return nil, returnErr
+	}
+	if response == nil {
+		returnErr = fmt.Errorf(
+			"Response from getNmAgentSupportedApis call is <nil>")
+		logger.Errorf("[Azure-CNS] %s", returnErr)
+		return nil, returnErr
+	}
+	if response.StatusCode != http.StatusOK {
 		returnErr = fmt.Errorf(
 			"Failed to retrieve Supported Apis from NMAgent with StatusCode: %d",
 			response.StatusCode)
-		if err != nil {
-			returnErr = fmt.Errorf(
-				"Failed to retrieve Supported Apis from NMAgent with error %v",
-				err.Error())
-		}
 		logger.Errorf("[Azure-CNS] %s", returnErr)
-		return []string{}, returnErr
+		return nil, returnErr
 	}
 
 	var xmlDoc NMAgentSupportedApisResponseXML
@@ -132,10 +140,10 @@ func GetNmAgentSupportedApis(
 			"Failed to decode XML response of Supported Apis from NMAgent with error %v",
 			err.Error())
 		logger.Errorf("[Azure-CNS] %s", returnErr)
-		return []string{}, returnErr
+		return nil, returnErr
 	}
 
-	logger.Printf("[NMAgentClient][Response] GetNmAgentSupportedApis. Response: %+v. Error: %v",
+	logger.Printf("[NMAgentClient][Response] GetNmAgentSupportedApis. Response: %+v.",
 		response, err)
 	return xmlDoc.SupportedApis, nil
 }
diff --git a/cns/restserver/api.go b/cns/restserver/api.go
index 35060b5392..7e7d335e76 100644
--- a/cns/restserver/api.go
+++ b/cns/restserver/api.go
@@ -1417,7 +1417,7 @@ func (service *HTTPRestService) deleteHostNCApipaEndpoint(w http.ResponseWriter,
 	logger.Response(service.Name, response, response.Response.ReturnCode, ReturnCodeToString(response.Response.ReturnCode), err)
 }
 
-// This function is used to query NMagents's supproted APIs list
+// This function is used to query NMagents's supported APIs list
 func (service *HTTPRestService) nmAgentSupportedApisHandler(w http.ResponseWriter, r *http.Request) {
 	logger.Request(service.Name, "nmAgentSupportedApisHandler", nil)
 	var (
@@ -1442,6 +1442,9 @@ func (service *HTTPRestService) nmAgentSupportedApisHandler(w http.ResponseWrite
 			returnCode = NmAgentSupportedApisError
 			returnMessage = fmt.Sprintf("[Azure-CNS] %s", retErr.Error())
 		}
+		if supportedApis == nil {
+			supportedApis = []string{}
+		}
 
 	default:
 		returnMessage = "[Azure-CNS] NmAgentSupported API list expects a POST method."
diff --git a/cns/restserver/internalapi.go b/cns/restserver/internalapi.go
index 3ca07137bf..b73ef44eda 100644
--- a/cns/restserver/internalapi.go
+++ b/cns/restserver/internalapi.go
@@ -266,16 +266,13 @@ func (service *HTTPRestService) CreateOrUpdateNetworkContainerInternal(req cns.C
 
 }
 
-// Try to register node with DNC when CNS is started in managed DNC mode
+// RegisterNode - Tries to register node with DNC when CNS is started in managed DNC mode
 func RegisterNode(httpc *http.Client, httpRestService cns.HTTPService, dncEP, infraVnet, nodeID string) error {
 	logger.Printf("[Azure CNS] Registering node %s with Infrastructure Network: %s PrivateEndpoint: %s", nodeID, infraVnet, dncEP)
 
 	var (
 		numCPU              = runtime.NumCPU()
 		url                 = fmt.Sprintf(common.RegisterNodeURLFmt, dncEP, infraVnet, nodeID, dncApiVersion)
-		response            *http.Response
-		err                 = fmt.Errorf("")
-		body                bytes.Buffer
 		nodeRegisterRequest cns.NodeRegisterRequest
 	)
 
@@ -283,40 +280,74 @@ func RegisterNode(httpc *http.Client, httpRestService cns.HTTPService, dncEP, in
 	supportedApis, retErr := nmagentclient.GetNmAgentSupportedApis(httpc, "")
 
 	if retErr != nil {
-		logger.Printf("[Azure CNS] Failed to retrieve SupportedApis from NMagent of node %s with Infrastructure Network: %s PrivateEndpoint: %s",
+		logger.Errorf("[Azure CNS] Failed to retrieve SupportedApis from NMagent of node %s with Infrastructure Network: %s PrivateEndpoint: %s",
 			nodeID, infraVnet, dncEP)
+		return retErr
+	}
+
+	//To avoid any null-pointer deferencing errors.
+	if supportedApis == nil {
+		supportedApis = []string{}
 	}
 
 	nodeRegisterRequest.NmAgentSupportedApis = supportedApis
-	if err := json.NewEncoder(&body).Encode(nodeRegisterRequest); err != nil {
+
+	nodeRegisterTicker := time.NewTicker(time.Duration(time.Second) * common.FiveSeconds)
+	responseChan := make(chan error)
+
+	for {
+		select {
+		case responseErr := <-responseChan:
+			return responseErr
+		case <-nodeRegisterTicker.C:
+			go sendRegisterNodeRequest(httpc, httpRestService, nodeRegisterRequest, url, responseChan)
+		}
+	}
+}
+
+// sendRegisterNodeRequest func helps in registering the node until there is an error.
+func sendRegisterNodeRequest(
+	httpc *http.Client,
+	httpRestService cns.HTTPService,
+	nodeRegisterRequest cns.NodeRegisterRequest,
+	registerURL string,
+	responseChan chan<- error) {
+
+	var (
+		body     bytes.Buffer
+		response *http.Response
+		err      = fmt.Errorf("")
+	)
+
+	err = json.NewEncoder(&body).Encode(nodeRegisterRequest)
+	if err != nil {
 		log.Errorf("encoding json failed with %v", err)
-		return err
+		responseChan <- err
+		return
 	}
 
-	for sleep := true; err != nil; sleep = true {
-		response, err = httpc.Post(url, "application/json", &body)
-		if err == nil {
-			if response.StatusCode == http.StatusCreated {
-				var req cns.SetOrchestratorTypeRequest
-				json.NewDecoder(response.Body).Decode(&req)
-				httpRestService.SetNodeOrchestrator(&req)
-				sleep = false
-			} else {
-				err = fmt.Errorf("[Azure CNS] Failed to register node with http status code %s", strconv.Itoa(response.StatusCode))
-				logger.Errorf(err.Error())
-				return err
+	response, err = httpc.Post(registerURL, "application/json", &body)
+	if err == nil {
+		if response.StatusCode == http.StatusCreated {
+			var req cns.SetOrchestratorTypeRequest
+			decodeErr := json.NewDecoder(response.Body).Decode(&req)
+			if decodeErr != nil {
+				log.Errorf("decoding Node Resgister response json failed with %v", err)
+				responseChan <- err
+				return
 			}
+			httpRestService.SetNodeOrchestrator(&req)
 
-			response.Body.Close()
+			logger.Printf("[Azure CNS] Node Registered")
+			responseChan <- nil
 		} else {
-			logger.Errorf("[Azure CNS] Failed to register node with err: %+v", err)
+			err = fmt.Errorf("[Azure CNS] Failed to register node with http status code %s", strconv.Itoa(response.StatusCode))
+			logger.Errorf(err.Error())
+			responseChan <- err
 		}
 
-		if sleep {
-			time.Sleep(common.FiveSeconds)
-		}
+		response.Body.Close()
+	} else {
+		logger.Errorf("[Azure CNS] Failed to register node with err: %+v", err)
 	}
-
-	logger.Printf("[Azure CNS] Node Registered")
-	return nil
 }

From 84742f89701a08433b88ee820d42ad654d4dd45b Mon Sep 17 00:00:00 2001
From: vakr <vakr@microsoft.com>
Date: Wed, 11 Nov 2020 16:55:00 -0800
Subject: [PATCH 12/19] resolving Merge conflict issues

---
 cns/NetworkContainerContract.go    | 8 ++++++--
 cns/nmagentclient/nmagentclient.go | 4 +---
 cns/restserver/const.go            | 2 +-
 cns/service/main.go                | 4 ++--
 4 files changed, 10 insertions(+), 8 deletions(-)

diff --git a/cns/NetworkContainerContract.go b/cns/NetworkContainerContract.go
index d7c41ca9f9..fde27bc3a5 100644
--- a/cns/NetworkContainerContract.go
+++ b/cns/NetworkContainerContract.go
@@ -374,7 +374,11 @@ func (networkContainerRequestPolicy *NetworkContainerRequestPolicies) Validate()
 // NodeInfoResponse - Struct to hold the node info response.
 type NodeInfoResponse struct {
 	NetworkContainers  []CreateNetworkContainerRequest
-	GetNCVersionURLFmt string
 	NmAgentApisMissing bool
-	NetworkContainers []CreateNetworkContainerRequest
+}
+
+// NodeRegisterRequest - Struct to hold the node register request.
+type NodeRegisterRequest struct {
+	NumCPU               int
+	NmAgentSupportedApis []string
 }
diff --git a/cns/nmagentclient/nmagentclient.go b/cns/nmagentclient/nmagentclient.go
index 1332086df7..719dabc3e9 100644
--- a/cns/nmagentclient/nmagentclient.go
+++ b/cns/nmagentclient/nmagentclient.go
@@ -16,12 +16,10 @@ const (
 	WireserverIP = "168.63.129.16"
 
 	//GetNmAgentSupportedApiURLFmt Api endpoint to get supported Apis of NMAgent
-	GetNmAgentSupportedApiURLFmt = "http://%s/machine/plugins/?comp=nmagent&type=GetSupportedApis"
+	GetNmAgentSupportedApiURLFmt     = "http://%s/machine/plugins/?comp=nmagent&type=GetSupportedApis"
 	GetNetworkContainerVersionURLFmt = "http://%s/machine/plugins/?comp=nmagent&type=NetworkManagement/interfaces/%s/networkContainers/%s/version/authenticationToken/%s/api-version/1"
 )
 
-var WireserverIP = "168.63.129.16"
-
 // NMANetworkContainerResponse - NMAgent response.
 type NMANetworkContainerResponse struct {
 	ResponseCode       string `json:"httpStatusCode"`
diff --git a/cns/restserver/const.go b/cns/restserver/const.go
index 1570a5f2f2..8cb329cf05 100644
--- a/cns/restserver/const.go
+++ b/cns/restserver/const.go
@@ -37,7 +37,7 @@ const (
 	UnsupportedOrchestratorContext         = 34
 	NetworkContainerVfpProgramComplete     = 35
 	NetworkContainerVfpProgramCheckSkipped = 36
-  NmAgentSupportedApisError              = 37
+	NmAgentSupportedApisError              = 37
 	UnexpectedError                        = 99
 )
 
diff --git a/cns/service/main.go b/cns/service/main.go
index 3484ab3a43..fd34799fd1 100644
--- a/cns/service/main.go
+++ b/cns/service/main.go
@@ -7,14 +7,14 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	localtls "github.com/Azure/azure-container-networking/server/tls"
-	"net/http"
 	"os"
 	"os/signal"
 	"strings"
 	"syscall"
 	"time"
 
+	localtls "github.com/Azure/azure-container-networking/server/tls"
+
 	"github.com/Azure/azure-container-networking/cns/ipampoolmonitor"
 
 	"github.com/Azure/azure-container-networking/aitelemetry"

From 3eedd84c86c9eb47cfa0f4504b07bba8b713fb81 Mon Sep 17 00:00:00 2001
From: vakr <vakr@microsoft.com>
Date: Thu, 12 Nov 2020 09:54:11 -0800
Subject: [PATCH 13/19] changing wireserver to Var, as tests are trying to
 change this value

---
 cns/nmagentclient/nmagentclient.go | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/cns/nmagentclient/nmagentclient.go b/cns/nmagentclient/nmagentclient.go
index 719dabc3e9..f587d5cac9 100644
--- a/cns/nmagentclient/nmagentclient.go
+++ b/cns/nmagentclient/nmagentclient.go
@@ -12,14 +12,14 @@ import (
 )
 
 const (
-	//WireServerIP - wire server ip
-	WireserverIP = "168.63.129.16"
-
 	//GetNmAgentSupportedApiURLFmt Api endpoint to get supported Apis of NMAgent
 	GetNmAgentSupportedApiURLFmt     = "http://%s/machine/plugins/?comp=nmagent&type=GetSupportedApis"
 	GetNetworkContainerVersionURLFmt = "http://%s/machine/plugins/?comp=nmagent&type=NetworkManagement/interfaces/%s/networkContainers/%s/version/authenticationToken/%s/api-version/1"
 )
 
+//WireServerIP - wire server ip
+var WireserverIP = "168.63.129.16"
+
 // NMANetworkContainerResponse - NMAgent response.
 type NMANetworkContainerResponse struct {
 	ResponseCode       string `json:"httpStatusCode"`

From b971ce473f0511c7e2c4cef143fc49854c9f06da Mon Sep 17 00:00:00 2001
From: vakr <vakr@microsoft.com>
Date: Thu, 12 Nov 2020 13:03:16 -0800
Subject: [PATCH 14/19] Correcting a timing of the ticker

---
 cns/restserver/internalapi.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cns/restserver/internalapi.go b/cns/restserver/internalapi.go
index 99add62313..5d35f25852 100644
--- a/cns/restserver/internalapi.go
+++ b/cns/restserver/internalapi.go
@@ -289,7 +289,7 @@ func RegisterNode(httpc *http.Client, httpRestService cns.HTTPService, dncEP, in
 
 	nodeRegisterRequest.NmAgentSupportedApis = supportedApis
 
-	nodeRegisterTicker := time.NewTicker(time.Duration(time.Second) * common.FiveSeconds)
+	nodeRegisterTicker := time.NewTicker(common.FiveSeconds)
 	responseChan := make(chan error)
 
 	for {

From 6717e485e76281da356ee5151393a049eeed029d Mon Sep 17 00:00:00 2001
From: vakr <vakr@microsoft.com>
Date: Thu, 12 Nov 2020 13:13:13 -0800
Subject: [PATCH 15/19] Correcting a timing of the ticker

---
 cns/nmagentclient/nmagentclient.go | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/cns/nmagentclient/nmagentclient.go b/cns/nmagentclient/nmagentclient.go
index f587d5cac9..5a1c00986f 100644
--- a/cns/nmagentclient/nmagentclient.go
+++ b/cns/nmagentclient/nmagentclient.go
@@ -144,7 +144,6 @@ func GetNmAgentSupportedApis(
 		return nil, returnErr
 	}
 
-	logger.Printf("[NMAgentClient][Response] GetNmAgentSupportedApis. Response: %+v.",
-		response, err)
+	logger.Printf("[NMAgentClient][Response] GetNmAgentSupportedApis. Response: %+v.", response)
 	return xmlDoc.SupportedApis, nil
 }

From 2129a6f5b17caefd89bd01466045e6d874432074 Mon Sep 17 00:00:00 2001
From: vakr <vakr@microsoft.com>
Date: Mon, 16 Nov 2020 10:08:34 -0800
Subject: [PATCH 16/19] Adjusting logic on re-registering the Node in mDNC case

---
 cns/NetworkContainerContract.go    |  3 +-
 cns/restserver/internalapi.go      | 97 ------------------------------
 cns/restserver/internalapi_test.go | 60 ------------------
 cns/service/main.go                | 93 +++++++++++++++++++++++++++-
 4 files changed, 93 insertions(+), 160 deletions(-)

diff --git a/cns/NetworkContainerContract.go b/cns/NetworkContainerContract.go
index fde27bc3a5..5d8ee0ca6e 100644
--- a/cns/NetworkContainerContract.go
+++ b/cns/NetworkContainerContract.go
@@ -373,8 +373,7 @@ func (networkContainerRequestPolicy *NetworkContainerRequestPolicies) Validate()
 
 // NodeInfoResponse - Struct to hold the node info response.
 type NodeInfoResponse struct {
-	NetworkContainers  []CreateNetworkContainerRequest
-	NmAgentApisMissing bool
+	NetworkContainers []CreateNetworkContainerRequest
 }
 
 // NodeRegisterRequest - Struct to hold the node register request.
diff --git a/cns/restserver/internalapi.go b/cns/restserver/internalapi.go
index 5d35f25852..9d2a4ffb3d 100644
--- a/cns/restserver/internalapi.go
+++ b/cns/restserver/internalapi.go
@@ -10,9 +10,6 @@ import (
 	"net/http"
 	"net/http/httptest"
 	"reflect"
-	"runtime"
-	"strconv"
-	"time"
 
 	"github.com/Azure/azure-container-networking/cns"
 	"github.com/Azure/azure-container-networking/cns/logger"
@@ -127,14 +124,6 @@ func (service *HTTPRestService) SyncNodeStatus(dncEP, infraVnet, nodeID string,
 	service.saveState()
 	service.Unlock()
 
-	if nodeInfoResponse.NmAgentApisMissing {
-		// RegisterNode again with NmAgent Apis list
-		retErr := RegisterNode(httpc, service, dncEP, infraVnet, nodeID)
-		if retErr != nil {
-			logger.Errorf("[Azure-CNS] Failed to register Node ID: %s with error: %s", nodeID, err.Error())
-		}
-	}
-
 	// delete dangling NCs
 	for nc := range ncsToBeDeleted {
 		var body bytes.Buffer
@@ -262,89 +251,3 @@ func (service *HTTPRestService) CreateOrUpdateNetworkContainerInternal(req cns.C
 
 	return returnCode
 }
-
-// RegisterNode - Tries to register node with DNC when CNS is started in managed DNC mode
-func RegisterNode(httpc *http.Client, httpRestService cns.HTTPService, dncEP, infraVnet, nodeID string) error {
-	logger.Printf("[Azure CNS] Registering node %s with Infrastructure Network: %s PrivateEndpoint: %s", nodeID, infraVnet, dncEP)
-
-	var (
-		numCPU              = runtime.NumCPU()
-		url                 = fmt.Sprintf(common.RegisterNodeURLFmt, dncEP, infraVnet, nodeID, dncApiVersion)
-		nodeRegisterRequest cns.NodeRegisterRequest
-	)
-
-	nodeRegisterRequest.NumCPU = numCPU
-	supportedApis, retErr := nmagentclient.GetNmAgentSupportedApis(httpc, "")
-
-	if retErr != nil {
-		logger.Errorf("[Azure CNS] Failed to retrieve SupportedApis from NMagent of node %s with Infrastructure Network: %s PrivateEndpoint: %s",
-			nodeID, infraVnet, dncEP)
-		return retErr
-	}
-
-	//To avoid any null-pointer deferencing errors.
-	if supportedApis == nil {
-		supportedApis = []string{}
-	}
-
-	nodeRegisterRequest.NmAgentSupportedApis = supportedApis
-
-	nodeRegisterTicker := time.NewTicker(common.FiveSeconds)
-	responseChan := make(chan error)
-
-	for {
-		select {
-		case responseErr := <-responseChan:
-			return responseErr
-		case <-nodeRegisterTicker.C:
-			go sendRegisterNodeRequest(httpc, httpRestService, nodeRegisterRequest, url, responseChan)
-		}
-	}
-}
-
-// sendRegisterNodeRequest func helps in registering the node until there is an error.
-func sendRegisterNodeRequest(
-	httpc *http.Client,
-	httpRestService cns.HTTPService,
-	nodeRegisterRequest cns.NodeRegisterRequest,
-	registerURL string,
-	responseChan chan<- error) {
-
-	var (
-		body     bytes.Buffer
-		response *http.Response
-		err      = fmt.Errorf("")
-	)
-
-	err = json.NewEncoder(&body).Encode(nodeRegisterRequest)
-	if err != nil {
-		log.Errorf("encoding json failed with %v", err)
-		responseChan <- err
-		return
-	}
-
-	response, err = httpc.Post(registerURL, "application/json", &body)
-	if err == nil {
-		if response.StatusCode == http.StatusCreated {
-			var req cns.SetOrchestratorTypeRequest
-			decodeErr := json.NewDecoder(response.Body).Decode(&req)
-			if decodeErr != nil {
-				log.Errorf("decoding Node Resgister response json failed with %v", err)
-				responseChan <- err
-				return
-			}
-			httpRestService.SetNodeOrchestrator(&req)
-
-			logger.Printf("[Azure CNS] Node Registered")
-			responseChan <- nil
-		} else {
-			err = fmt.Errorf("[Azure CNS] Failed to register node with http status code %s", strconv.Itoa(response.StatusCode))
-			logger.Errorf(err.Error())
-			responseChan <- err
-		}
-
-		response.Body.Close()
-	} else {
-		logger.Errorf("[Azure CNS] Failed to register node with err: %+v", err)
-	}
-}
diff --git a/cns/restserver/internalapi_test.go b/cns/restserver/internalapi_test.go
index e1af381f18..843f0a98a7 100644
--- a/cns/restserver/internalapi_test.go
+++ b/cns/restserver/internalapi_test.go
@@ -4,15 +4,10 @@
 package restserver
 
 import (
-	"bytes"
 	"encoding/json"
 	"fmt"
-	"io"
-	"io/ioutil"
-	"net/http"
 	"reflect"
 	"strconv"
-	"strings"
 	"testing"
 
 	"github.com/Azure/azure-container-networking/cns"
@@ -221,17 +216,6 @@ func TestReconcileNCWithSystemPods(t *testing.T) {
 	validateNCStateAfterReconcile(t, &req, expectedNcCount, expectedAllocatedPods)
 }
 
-func TestRegisterNode(t *testing.T) {
-	restartService()
-	setEnv(t)
-	setOrchestratorTypeInternal(cns.KubernetesCRD)
-
-	err := RegisterNode(NewTestClient(), svc, "localhost", "dummyvnet", "dummyNodeId")
-	if err != nil {
-		t.Errorf("Unexpected failure on register Node %s", err)
-	}
-}
-
 func setOrchestratorTypeInternal(orchestratorType string) {
 	fmt.Println("setOrchestratorTypeInternal")
 	svc.state.OrchestratorType = orchestratorType
@@ -470,47 +454,3 @@ func restartService() {
 	service.Stop()
 	startService()
 }
-
-// RoundTripFunc .
-type RoundTripFunc func(req *http.Request) *http.Response
-
-// RoundTrip .
-func (f RoundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) {
-	return f(req), nil
-}
-
-func mockRountTrip(req *http.Request) *http.Response {
-	var (
-		body       io.ReadCloser
-		returnCode = 200
-	)
-	// Test request parameters
-	switch {
-	case strings.Contains(req.URL.String(), "GetSupportedApis"):
-		// Handle Call to NMAgent
-		body = ioutil.NopCloser(bytes.NewBufferString(hostSupportedApis))
-
-	case strings.Contains(req.URL.String(), "dummyNodeId"):
-		//Handle Call to register Node
-		body = ioutil.NopCloser(bytes.NewBufferString("OK"))
-		returnCode = 201
-
-	default:
-		returnCode = 200
-	}
-
-	return &http.Response{
-		StatusCode: returnCode,
-		// Send response to be tested
-		Body: body,
-		// Must be set to non-nil value or it panics
-		Header: make(http.Header),
-	}
-}
-
-//NewTestClient returns *http.Client with Transport replaced to avoid making real calls
-func NewTestClient() *http.Client {
-	return &http.Client{
-		Transport: RoundTripFunc(mockRountTrip),
-	}
-}
diff --git a/cns/service/main.go b/cns/service/main.go
index fd34799fd1..d63a5bc685 100644
--- a/cns/service/main.go
+++ b/cns/service/main.go
@@ -4,11 +4,15 @@
 package main
 
 import (
+	"bytes"
 	"context"
 	"encoding/json"
 	"fmt"
+	"net/http"
 	"os"
 	"os/signal"
+	"runtime"
+	"strconv"
 	"strings"
 	"syscall"
 	"time"
@@ -16,6 +20,7 @@ import (
 	localtls "github.com/Azure/azure-container-networking/server/tls"
 
 	"github.com/Azure/azure-container-networking/cns/ipampoolmonitor"
+	"github.com/Azure/azure-container-networking/cns/nmagentclient"
 
 	"github.com/Azure/azure-container-networking/aitelemetry"
 	"github.com/Azure/azure-container-networking/cnm/ipam"
@@ -243,6 +248,92 @@ func printVersion() {
 	fmt.Printf("Version %v\n", version)
 }
 
+// RegisterNode - Tries to register node with DNC when CNS is started in managed DNC mode
+func registerNode(httpc *http.Client, httpRestService cns.HTTPService, dncEP, infraVnet, nodeID string) error {
+	logger.Printf("[Azure CNS] Registering node %s with Infrastructure Network: %s PrivateEndpoint: %s", nodeID, infraVnet, dncEP)
+
+	var (
+		numCPU              = runtime.NumCPU()
+		url                 = fmt.Sprintf(acn.RegisterNodeURLFmt, dncEP, infraVnet, nodeID, dncApiVersion)
+		nodeRegisterRequest cns.NodeRegisterRequest
+	)
+
+	nodeRegisterRequest.NumCPU = numCPU
+	supportedApis, retErr := nmagentclient.GetNmAgentSupportedApis(httpc, "")
+
+	if retErr != nil {
+		logger.Errorf("[Azure CNS] Failed to retrieve SupportedApis from NMagent of node %s with Infrastructure Network: %s PrivateEndpoint: %s",
+			nodeID, infraVnet, dncEP)
+		return retErr
+	}
+
+	//To avoid any null-pointer deferencing errors.
+	if supportedApis == nil {
+		supportedApis = []string{}
+	}
+
+	nodeRegisterRequest.NmAgentSupportedApis = supportedApis
+
+	nodeRegisterTicker := time.NewTicker(acn.FiveSeconds)
+	responseChan := make(chan error)
+
+	for {
+		select {
+		case responseErr := <-responseChan:
+			return responseErr
+		case <-nodeRegisterTicker.C:
+			go sendRegisterNodeRequest(httpc, httpRestService, nodeRegisterRequest, url, responseChan)
+		}
+	}
+}
+
+// sendRegisterNodeRequest func helps in registering the node until there is an error.
+func sendRegisterNodeRequest(
+	httpc *http.Client,
+	httpRestService cns.HTTPService,
+	nodeRegisterRequest cns.NodeRegisterRequest,
+	registerURL string,
+	responseChan chan<- error) {
+
+	var (
+		body     bytes.Buffer
+		response *http.Response
+		err      = fmt.Errorf("")
+	)
+
+	err = json.NewEncoder(&body).Encode(nodeRegisterRequest)
+	if err != nil {
+		log.Errorf("encoding json failed with %v", err)
+		responseChan <- err
+		return
+	}
+
+	response, err = httpc.Post(registerURL, "application/json", &body)
+	if err == nil {
+		if response.StatusCode == http.StatusCreated {
+			var req cns.SetOrchestratorTypeRequest
+			decodeErr := json.NewDecoder(response.Body).Decode(&req)
+			if decodeErr != nil {
+				log.Errorf("decoding Node Resgister response json failed with %v", err)
+				responseChan <- err
+				return
+			}
+			httpRestService.SetNodeOrchestrator(&req)
+
+			logger.Printf("[Azure CNS] Node Registered")
+			responseChan <- nil
+		} else {
+			err = fmt.Errorf("[Azure CNS] Failed to register node with http status code %s", strconv.Itoa(response.StatusCode))
+			logger.Errorf(err.Error())
+			responseChan <- err
+		}
+
+		response.Body.Close()
+	} else {
+		logger.Errorf("[Azure CNS] Failed to register node with err: %+v", err)
+	}
+}
+
 // Main is the entry point for CNS.
 func main() {
 	// Initialize and parse command line arguments.
@@ -418,7 +509,7 @@ func main() {
 		httpRestService.SetOption(acn.OptInfrastructureNetworkID, infravnet)
 		httpRestService.SetOption(acn.OptNodeID, nodeID)
 
-		registerErr := restserver.RegisterNode(acn.GetHttpClient(), httpRestService, privateEndpoint, infravnet, nodeID)
+		registerErr := registerNode(acn.GetHttpClient(), httpRestService, privateEndpoint, infravnet, nodeID)
 		if registerErr != nil {
 			logger.Errorf("[Azure CNS] Resgistering Node failed with error: %v PrivateEndpoint: %s InfrastructureNetworkID: %s NodeID: %s",
 				registerErr,

From aafc04124f6f763926c682241adda4715a955f82 Mon Sep 17 00:00:00 2001
From: vakr <vakr@microsoft.com>
Date: Tue, 17 Nov 2020 19:47:21 -0800
Subject: [PATCH 17/19] Addressing comments

---
 cns/nmagentclient/nmagentclient.go |  2 ++
 cns/service/main.go                | 44 ++++++++++++++++--------------
 2 files changed, 25 insertions(+), 21 deletions(-)

diff --git a/cns/nmagentclient/nmagentclient.go b/cns/nmagentclient/nmagentclient.go
index 5a1c00986f..e254a09547 100644
--- a/cns/nmagentclient/nmagentclient.go
+++ b/cns/nmagentclient/nmagentclient.go
@@ -125,6 +125,8 @@ func GetNmAgentSupportedApis(
 		logger.Errorf("[Azure-CNS] %s", returnErr)
 		return nil, returnErr
 	}
+	defer response.Body.Close()
+
 	if response.StatusCode != http.StatusOK {
 		returnErr = fmt.Errorf(
 			"Failed to retrieve Supported Apis from NMAgent with StatusCode: %d",
diff --git a/cns/service/main.go b/cns/service/main.go
index d63a5bc685..aaf1fdf8c3 100644
--- a/cns/service/main.go
+++ b/cns/service/main.go
@@ -278,11 +278,12 @@ func registerNode(httpc *http.Client, httpRestService cns.HTTPService, dncEP, in
 	responseChan := make(chan error)
 
 	for {
+		sendRegisterNodeRequest(httpc, httpRestService, nodeRegisterRequest, url, responseChan)
 		select {
 		case responseErr := <-responseChan:
 			return responseErr
 		case <-nodeRegisterTicker.C:
-			go sendRegisterNodeRequest(httpc, httpRestService, nodeRegisterRequest, url, responseChan)
+			continue
 		}
 	}
 }
@@ -309,29 +310,30 @@ func sendRegisterNodeRequest(
 	}
 
 	response, err = httpc.Post(registerURL, "application/json", &body)
-	if err == nil {
-		if response.StatusCode == http.StatusCreated {
-			var req cns.SetOrchestratorTypeRequest
-			decodeErr := json.NewDecoder(response.Body).Decode(&req)
-			if decodeErr != nil {
-				log.Errorf("decoding Node Resgister response json failed with %v", err)
-				responseChan <- err
-				return
-			}
-			httpRestService.SetNodeOrchestrator(&req)
+	if err != nil {
+		logger.Errorf("[Azure CNS] Failed to register node with err: %+v", err)
+		return
+	}
+	defer response.Body.Close()
 
-			logger.Printf("[Azure CNS] Node Registered")
-			responseChan <- nil
-		} else {
-			err = fmt.Errorf("[Azure CNS] Failed to register node with http status code %s", strconv.Itoa(response.StatusCode))
-			logger.Errorf(err.Error())
-			responseChan <- err
-		}
+	if response.StatusCode != http.StatusCreated {
+		err = fmt.Errorf("[Azure CNS] Failed to register node with http status code %s", strconv.Itoa(response.StatusCode))
+		logger.Errorf(err.Error())
+		responseChan <- err
+		return
+	}
 
-		response.Body.Close()
-	} else {
-		logger.Errorf("[Azure CNS] Failed to register node with err: %+v", err)
+	var req cns.SetOrchestratorTypeRequest
+	decodeErr := json.NewDecoder(response.Body).Decode(&req)
+	if decodeErr != nil {
+		log.Errorf("decoding Node Resgister response json failed with %v", err)
+		responseChan <- err
+		return
 	}
+	httpRestService.SetNodeOrchestrator(&req)
+
+	logger.Printf("[Azure CNS] Node Registered")
+	responseChan <- nil
 }
 
 // Main is the entry point for CNS.

From cf34795529706d0303aff226f029bdbf1606fd90 Mon Sep 17 00:00:00 2001
From: vakr <vakr@microsoft.com>
Date: Thu, 19 Nov 2020 11:48:02 -0800
Subject: [PATCH 18/19] changing logic around noderegister

---
 cns/service/main.go | 53 +++++++++++++++++++++++----------------------
 1 file changed, 27 insertions(+), 26 deletions(-)

diff --git a/cns/service/main.go b/cns/service/main.go
index aaf1fdf8c3..c5ca45ac45 100644
--- a/cns/service/main.go
+++ b/cns/service/main.go
@@ -49,6 +49,9 @@ const (
 	configFileName                    = "config.json"
 	dncApiVersion                     = "?api-version=2018-03-01"
 	poolIPAMRefreshRateInMilliseconds = 1000
+
+	// 720 * acn.FiveSeconds sec sleeps = 1Hr
+	maxRetryNodeRegister = 720
 )
 
 // Version is populated by make during build.
@@ -274,18 +277,20 @@ func registerNode(httpc *http.Client, httpRestService cns.HTTPService, dncEP, in
 
 	nodeRegisterRequest.NmAgentSupportedApis = supportedApis
 
-	nodeRegisterTicker := time.NewTicker(acn.FiveSeconds)
-	responseChan := make(chan error)
-
-	for {
-		sendRegisterNodeRequest(httpc, httpRestService, nodeRegisterRequest, url, responseChan)
-		select {
-		case responseErr := <-responseChan:
-			return responseErr
-		case <-nodeRegisterTicker.C:
-			continue
+	//CNS tries to register Node for maximum of an hour.
+	for tryNum := 0; tryNum <= maxRetryNodeRegister; tryNum++ {
+		success, err := sendRegisterNodeRequest(httpc, httpRestService, nodeRegisterRequest, url)
+		if err != nil {
+			return err
 		}
+		if success {
+			return nil
+		}
+		time.Sleep(acn.FiveSeconds)
 	}
+	logger.Errorf("[Azure CNS] Failed to register node %s after maximum reties for an hour with Infrastructure Network: %s PrivateEndpoint: %s",
+		nodeID, infraVnet, dncEP)
+	return nil
 }
 
 // sendRegisterNodeRequest func helps in registering the node until there is an error.
@@ -293,8 +298,7 @@ func sendRegisterNodeRequest(
 	httpc *http.Client,
 	httpRestService cns.HTTPService,
 	nodeRegisterRequest cns.NodeRegisterRequest,
-	registerURL string,
-	responseChan chan<- error) {
+	registerURL string) (bool, error) {
 
 	var (
 		body     bytes.Buffer
@@ -304,36 +308,33 @@ func sendRegisterNodeRequest(
 
 	err = json.NewEncoder(&body).Encode(nodeRegisterRequest)
 	if err != nil {
-		log.Errorf("encoding json failed with %v", err)
-		responseChan <- err
-		return
+		log.Errorf("[Azure CNS] Failed to register node while encoding json failed with non-retriable err %v", err)
+		return false, err
 	}
 
 	response, err = httpc.Post(registerURL, "application/json", &body)
 	if err != nil {
-		logger.Errorf("[Azure CNS] Failed to register node with err: %+v", err)
-		return
+		logger.Errorf("[Azure CNS] Failed to register node with retriable err: %+v", err)
+		return false, nil
 	}
 	defer response.Body.Close()
 
 	if response.StatusCode != http.StatusCreated {
-		err = fmt.Errorf("[Azure CNS] Failed to register node with http status code %s", strconv.Itoa(response.StatusCode))
+		err = fmt.Errorf("[Azure CNS] Failed to register node, DNC replied with http status code %s", strconv.Itoa(response.StatusCode))
 		logger.Errorf(err.Error())
-		responseChan <- err
-		return
+		return false, err
 	}
 
 	var req cns.SetOrchestratorTypeRequest
-	decodeErr := json.NewDecoder(response.Body).Decode(&req)
-	if decodeErr != nil {
-		log.Errorf("decoding Node Resgister response json failed with %v", err)
-		responseChan <- err
-		return
+	err = json.NewDecoder(response.Body).Decode(&req)
+	if err != nil {
+		log.Errorf("[Azure CNS] decoding Node Resgister response json failed with non-retriable err %v", err)
+		return false, err
 	}
 	httpRestService.SetNodeOrchestrator(&req)
 
 	logger.Printf("[Azure CNS] Node Registered")
-	responseChan <- nil
+	return true, nil
 }
 
 // Main is the entry point for CNS.

From 4d3de913160e3837bb03790a2d630535010d3aff Mon Sep 17 00:00:00 2001
From: vakr <vakr@microsoft.com>
Date: Thu, 19 Nov 2020 12:27:53 -0800
Subject: [PATCH 19/19] Addressing comments

---
 cns/service/main.go | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/cns/service/main.go b/cns/service/main.go
index c5ca45ac45..0262232be9 100644
--- a/cns/service/main.go
+++ b/cns/service/main.go
@@ -288,9 +288,8 @@ func registerNode(httpc *http.Client, httpRestService cns.HTTPService, dncEP, in
 		}
 		time.Sleep(acn.FiveSeconds)
 	}
-	logger.Errorf("[Azure CNS] Failed to register node %s after maximum reties for an hour with Infrastructure Network: %s PrivateEndpoint: %s",
+	return fmt.Errorf("[Azure CNS] Failed to register node %s after maximum reties for an hour with Infrastructure Network: %s PrivateEndpoint: %s",
 		nodeID, infraVnet, dncEP)
-	return nil
 }
 
 // sendRegisterNodeRequest func helps in registering the node until there is an error.
@@ -322,14 +321,14 @@ func sendRegisterNodeRequest(
 	if response.StatusCode != http.StatusCreated {
 		err = fmt.Errorf("[Azure CNS] Failed to register node, DNC replied with http status code %s", strconv.Itoa(response.StatusCode))
 		logger.Errorf(err.Error())
-		return false, err
+		return false, nil
 	}
 
 	var req cns.SetOrchestratorTypeRequest
 	err = json.NewDecoder(response.Body).Decode(&req)
 	if err != nil {
-		log.Errorf("[Azure CNS] decoding Node Resgister response json failed with non-retriable err %v", err)
-		return false, err
+		log.Errorf("[Azure CNS] decoding Node Resgister response json failed with err %v", err)
+		return false, nil
 	}
 	httpRestService.SetNodeOrchestrator(&req)