From b88dc059d4deafea6222db551b37db62398171cf Mon Sep 17 00:00:00 2001
From: neaggarw <neaggarw@microsoft.com>
Date: Mon, 10 May 2021 12:10:16 -0700
Subject: [PATCH 1/2] Fix to bubble up the error and restart in case it fails
 to create CRD controllers

---
 cns/service/main.go | 33 +++++++++++++++++++++++----------
 1 file changed, 23 insertions(+), 10 deletions(-)

diff --git a/cns/service/main.go b/cns/service/main.go
index b0d72cb139..e038f983a3 100644
--- a/cns/service/main.go
+++ b/cns/service/main.go
@@ -504,7 +504,11 @@ func main() {
 	if config.ChannelMode == cns.CRD {
 		requestControllerStopChannel := make(chan struct{})
 		defer close(requestControllerStopChannel)
-		IniitalizeCRDState(httpRestService, cnsconfig, requestControllerStopChannel)
+		err = IniitalizeCRDState(httpRestService, cnsconfig, requestControllerStopChannel)
+		if err != nil {
+			logger.Errorf("Failed to start CRD Controller, err:%v.\n", err)
+			return
+		}
 	}
 
 	logger.Printf("[Azure CNS] Start HTTP listener")
@@ -667,7 +671,7 @@ func main() {
 }
 
 // initializeCRD state
-func IniitalizeCRDState(httpRestService cns.HTTPService, cnsconfig configuration.CNSConfig, exitChan <-chan struct{}) {
+func IniitalizeCRDState(httpRestService cns.HTTPService, cnsconfig configuration.CNSConfig, exitChan <-chan struct{}) error {
 	var requestController requestcontroller.RequestController
 
 	logger.Printf("[Azure CNS] Starting request controller")
@@ -675,14 +679,14 @@ func IniitalizeCRDState(httpRestService cns.HTTPService, cnsconfig configuration
 	kubeConfig, err := kubecontroller.GetKubeConfig()
 	if err != nil {
 		logger.Errorf("[Azure CNS] Failed to get kubeconfig for request controller: %v", err)
-		return
+		return err
 	}
 
 	//convert interface type to implementation type
 	httpRestServiceImplementation, ok := httpRestService.(*restserver.HTTPRestService)
 	if !ok {
 		logger.Errorf("[Azure CNS] Failed to convert interface httpRestService to implementation: %v", httpRestService)
-		return
+		return err
 	}
 
 	// Set orchestrator type
@@ -695,7 +699,7 @@ func IniitalizeCRDState(httpRestService cns.HTTPService, cnsconfig configuration
 	requestController, err = kubecontroller.NewCrdRequestController(httpRestServiceImplementation, kubeConfig)
 	if err != nil {
 		logger.Errorf("[Azure CNS] Failed to make crd request controller :%v", err)
-		return
+		return err
 	}
 
     // initialize the ipam pool monitor
@@ -704,7 +708,7 @@ func IniitalizeCRDState(httpRestService cns.HTTPService, cnsconfig configuration
 	err = requestController.InitRequestController()
 	if err != nil {
 		logger.Errorf("[Azure CNS] Failed to initialized cns state :%v", err)
-		return
+		return err
 	}
 
 	//Start the RequestController which starts the reconcile loop
@@ -737,11 +741,18 @@ func IniitalizeCRDState(httpRestService cns.HTTPService, cnsconfig configuration
 	ctx := context.Background()
 	logger.Printf("Starting IPAM Pool Monitor")
 	go func() {
-		if err := httpRestServiceImplementation.IPAMPoolMonitor.Start(ctx, poolIPAMRefreshRateInMilliseconds); err != nil {
-			logger.Errorf("[Azure CNS] Failed to start pool monitor with err: %v", err)
-		}
+		for {
+			if err := httpRestServiceImplementation.IPAMPoolMonitor.Start(ctx, poolIPAMRefreshRateInMilliseconds); err != nil {
+				logger.Errorf("[Azure CNS] Failed to start pool monitor with err: %v", err)
+				// todo: add a CNS metric to count # of failures
+			} else {
+				logger.Printf("[Azure CNS] Exiting IPAM Pool Monitor")
+				return
+			}
 
-		logger.Printf("[Azure CNS] Exiting IPAM Pool Monitor")
+			// Retry after 1sec
+			time.Sleep(time.Second)
+		}
 	}()
 
 	logger.Printf("Starting SyncHostNCVersion")
@@ -755,4 +766,6 @@ func IniitalizeCRDState(httpRestService cns.HTTPService, cnsconfig configuration
 
 		logger.Printf("[Azure CNS] Exiting SyncHostNCVersion")
 	}()
+
+	return nil
 }

From b91cec29827efae3a7463c577cb799b3cfde8b56 Mon Sep 17 00:00:00 2001
From: neaggarw <neaggarw@microsoft.com>
Date: Mon, 10 May 2021 12:36:51 -0700
Subject: [PATCH 2/2] Fix returned error

---
 cns/service/main.go | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/cns/service/main.go b/cns/service/main.go
index e038f983a3..0a73c48475 100644
--- a/cns/service/main.go
+++ b/cns/service/main.go
@@ -686,7 +686,8 @@ func IniitalizeCRDState(httpRestService cns.HTTPService, cnsconfig configuration
 	httpRestServiceImplementation, ok := httpRestService.(*restserver.HTTPRestService)
 	if !ok {
 		logger.Errorf("[Azure CNS] Failed to convert interface httpRestService to implementation: %v", httpRestService)
-		return err
+		return  fmt.Errorf("[Azure CNS] Failed to convert interface httpRestService to implementation: %v",
+			httpRestService)
 	}
 
 	// Set orchestrator type