Fix init DB conflict with multiple replicas

Signed-off-by: astraw99 <wangchengiscool@gmail.com>

workflow/controller/config.go

@@ -27,6 +27,20 @@ func (wfc *WorkflowController) updateConfig() error {
 	wfc.offloadNodeStatusRepo = sqldb.ExplosiveOffloadNodeStatusRepo
 	wfc.wfArchive = sqldb.NullWorkflowArchive
 	wfc.archiveLabelSelector = labels.Everything()
+
+	wfc.hydrator = hydrator.New(wfc.offloadNodeStatusRepo)
+	wfc.updateEstimatorFactory()
+	wfc.rateLimiter = wfc.newRateLimiter()
+
+	log.WithField("executorImage", wfc.executorImage()).
+		WithField("executorImagePullPolicy", wfc.executorImagePullPolicy()).
+		WithField("managedNamespace", wfc.GetManagedNamespace()).
+		Info()
+	return nil
+}
+
+// initDB inits argo DB tables
+func (wfc *WorkflowController) initDB() error {
 	persistence := wfc.Config.Persistence
 	if persistence != nil {
 		log.Info("Persistence configuration enabled")
@@ -75,14 +89,7 @@ func (wfc *WorkflowController) updateConfig() error {
 	} else {
 		log.Info("Persistence configuration disabled")
 	}
-	wfc.hydrator = hydrator.New(wfc.offloadNodeStatusRepo)
-	wfc.updateEstimatorFactory()
-	wfc.rateLimiter = wfc.newRateLimiter()
 
-	log.WithField("executorImage", wfc.executorImage()).
-		WithField("executorImagePullPolicy", wfc.executorImagePullPolicy()).
-		WithField("managedNamespace", wfc.GetManagedNamespace()).
-		Info()
 	return nil
 }
 

workflow/controller/controller.go

@@ -240,6 +240,11 @@ var indexers = cache.Indexers{
 func (wfc *WorkflowController) Run(ctx context.Context, wfWorkers, workflowTTLWorkers, podCleanupWorkers, cronWorkflowWorkers int) {
 	defer runtimeutil.HandleCrash(runtimeutil.PanicHandlers...)
 
+	// init DB after leader election (if enabled)
+	if err := wfc.initDB(); err != nil {
+		log.Fatalf("Failed to init db: %v", err)
+	}
+
 	ctx, cancel := context.WithCancel(ctx)
 	defer cancel()