feat: add --app-hard-resync flag to controller (#8928)

* feat: add --app-hard-resync flag to controller Signed-off-by: darshanime <deathbullet@gmail.com> * add to approvers Signed-off-by: pashavictorovich <pavel@codefresh.io> * Merge branch 'master' of github.com:pasha-codefresh/argo-cd into hard-refresh Signed-off-by: pashavictorovich <pavel@codefresh.io> # Conflicts: # cmd/argocd-application-controller/main.go # controller/appcontroller.go * merge Signed-off-by: pashavictorovich <pavel@codefresh.io> * merge Signed-off-by: pashavictorovich <pavel@codefresh.io> * regenerate docs Signed-off-by: pashavictorovich <pavel@codefresh.io> * regenerate docs Signed-off-by: pashavictorovich <pavel@codefresh.io> * fix tests Signed-off-by: pashavictorovich <pavel@codefresh.io> * read variable from cm Signed-off-by: pashavictorovich <pavel@codefresh.io> * reconciliation variable Signed-off-by: pashavictorovich <pavel@codefresh.io> * manifest Signed-off-by: pashavictorovich <pavel@codefresh.io> Co-authored-by: darshanime <deathbullet@gmail.com>
argoproj · Apr 4, 2022 · f11da56 · f11da56
1 parent 27d4edd
commit f11da56
Show file tree

Hide file tree

Showing 10 changed files with 89 additions and 13 deletions.
diff --git a/cmd/argocd-application-controller/commands/argocd_application_controller.go b/cmd/argocd-application-controller/commands/argocd_application_controller.go
@@ -35,12 +35,15 @@ const (
 	cliName = "argocd-application-controller"
 	// Default time in seconds for application resync period
 	defaultAppResyncPeriod = 180
+	// Default time in seconds for application hard resync period
+	defaultAppHardResyncPeriod = 0
 )
 
 func NewCommand() *cobra.Command {
 	var (
 		clientConfig             clientcmd.ClientConfig
 		appResyncPeriod          int64
+		appHardResyncPeriod      int64
 		repoServerAddress        string
 		repoServerTimeoutSeconds int
 		selfHealTimeoutSeconds   int
@@ -78,6 +81,8 @@ func NewCommand() *cobra.Command {
 			namespace, _, err := clientConfig.Namespace()
 			errors.CheckError(err)
 
+			hardResyncDuration := time.Duration(appHardResyncPeriod) * time.Second
+
 			var resyncDuration time.Duration
 			if appResyncPeriod == 0 {
 				// Re-sync should be disabled if period is 0. Set duration to a very long duration
@@ -129,6 +134,7 @@ func NewCommand() *cobra.Command {
 				cache,
 				kubectl,
 				resyncDuration,
+				hardResyncDuration,
 				time.Duration(selfHealTimeoutSeconds)*time.Second,
 				metricsPort,
 				metricsCacheExpiration,
@@ -152,6 +158,7 @@ func NewCommand() *cobra.Command {
 
 	clientConfig = cli.AddKubectlFlagsToCmd(&command)
 	command.Flags().Int64Var(&appResyncPeriod, "app-resync", int64(env.ParseDurationFromEnv("ARGOCD_RECONCILIATION_TIMEOUT", defaultAppResyncPeriod*time.Second, 0, math.MaxInt64).Seconds()), "Time period in seconds for application resync.")
+	command.Flags().Int64Var(&appHardResyncPeriod, "app-hard-resync", int64(env.ParseDurationFromEnv("ARGOCD_HARD_RECONCILIATION_TIMEOUT", defaultAppHardResyncPeriod*time.Second, 0, math.MaxInt64).Seconds()), "Time period in seconds for application hard resync.")
 	command.Flags().StringVar(&repoServerAddress, "repo-server", env.StringFromEnv("ARGOCD_APPLICATION_CONTROLLER_REPO_SERVER", common.DefaultRepoServerAddr), "Repo server address.")
 	command.Flags().IntVar(&repoServerTimeoutSeconds, "repo-server-timeout-seconds", env.ParseNumFromEnv("ARGOCD_APPLICATION_CONTROLLER_REPO_SERVER_TIMEOUT_SECONDS", 60, 0, math.MaxInt64), "Repo server RPC call timeout seconds.")
 	command.Flags().IntVar(&statusProcessors, "status-processors", env.ParseNumFromEnv("ARGOCD_APPLICATION_CONTROLLER_STATUS_PROCESSORS", 20, 0, math.MaxInt32), "Number of application status processors")

diff --git a/controller/appcontroller.go b/controller/appcontroller.go
@@ -102,6 +102,7 @@ type ApplicationController struct {
 	appStateManager               AppStateManager
 	stateCache                    statecache.LiveStateCache
 	statusRefreshTimeout          time.Duration
+	statusHardRefreshTimeout      time.Duration
 	selfHealTimeout               time.Duration
 	repoClientset                 apiclient.Clientset
 	db                            db.ArgoDB
@@ -124,14 +125,15 @@ func NewApplicationController(
 	argoCache *appstatecache.Cache,
 	kubectl kube.Kubectl,
 	appResyncPeriod time.Duration,
+	appHardResyncPeriod time.Duration,
 	selfHealTimeout time.Duration,
 	metricsPort int,
 	metricsCacheExpiration time.Duration,
 	metricsApplicationLabels []string,
 	kubectlParallelismLimit int64,
 	clusterFilter func(cluster *appv1.Cluster) bool,
 ) (*ApplicationController, error) {
-	log.Infof("appResyncPeriod=%v", appResyncPeriod)
+	log.Infof("appResyncPeriod=%v, appHardResyncPeriod=%v", appResyncPeriod, appHardResyncPeriod)
 	db := db.NewDB(namespace, settingsMgr, kubeClientset)
 	ctrl := ApplicationController{
 		cache:                         argoCache,
@@ -146,6 +148,7 @@ func NewApplicationController(
 		appComparisonTypeRefreshQueue: workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter()),
 		db:                            db,
 		statusRefreshTimeout:          appResyncPeriod,
+		statusHardRefreshTimeout:      appHardResyncPeriod,
 		refreshRequestedApps:          make(map[string]CompareWith),
 		refreshRequestedAppsMutex:     &sync.Mutex{},
 		auditLogger:                   argo.NewAuditLogger(namespace, kubeClientset, "argocd-application-controller"),
@@ -1253,7 +1256,7 @@ func (ctrl *ApplicationController) processAppRefreshQueueItem() (processNext boo
 		return
 	}
 	origApp = origApp.DeepCopy()
-	needRefresh, refreshType, comparisonLevel := ctrl.needRefreshAppStatus(origApp, ctrl.statusRefreshTimeout)
+	needRefresh, refreshType, comparisonLevel := ctrl.needRefreshAppStatus(origApp, ctrl.statusRefreshTimeout, ctrl.statusHardRefreshTimeout)
 
 	if !needRefresh {
 		return
@@ -1374,12 +1377,13 @@ func resourceStatusKey(res appv1.ResourceStatus) string {
 // Returns true if application never been compared, has changed or comparison result has expired.
 // Additionally returns whether full refresh was requested or not.
 // If full refresh is requested then target and live state should be reconciled, else only live state tree should be updated.
-func (ctrl *ApplicationController) needRefreshAppStatus(app *appv1.Application, statusRefreshTimeout time.Duration) (bool, appv1.RefreshType, CompareWith) {
+func (ctrl *ApplicationController) needRefreshAppStatus(app *appv1.Application, statusRefreshTimeout, statusHardRefreshTimeout time.Duration) (bool, appv1.RefreshType, CompareWith) {
 	logCtx := log.WithFields(log.Fields{"application": app.Name})
 	var reason string
 	compareWith := CompareWithLatest
 	refreshType := appv1.RefreshTypeNormal
-	expired := app.Status.ReconciledAt == nil || app.Status.ReconciledAt.Add(statusRefreshTimeout).Before(time.Now().UTC())
+	softExpired := app.Status.ReconciledAt == nil || app.Status.ReconciledAt.Add(statusRefreshTimeout).Before(time.Now().UTC())
+	hardExpired := (app.Status.ReconciledAt == nil || app.Status.ReconciledAt.Add(statusHardRefreshTimeout).Before(time.Now().UTC())) && statusHardRefreshTimeout.Seconds() != 0
 
 	if requestedType, ok := app.IsRefreshRequested(); ok {
 		compareWith = CompareWithLatestForceResolve
@@ -1389,15 +1393,19 @@ func (ctrl *ApplicationController) needRefreshAppStatus(app *appv1.Application,
 	} else if !app.Spec.Source.Equals(app.Status.Sync.ComparedTo.Source) {
 		reason = "spec.source differs"
 		compareWith = CompareWithLatestForceResolve
-	} else if expired {
+	} else if hardExpired || softExpired {
 		// The commented line below mysteriously crashes if app.Status.ReconciledAt is nil
 		// reason = fmt.Sprintf("comparison expired. reconciledAt: %v, expiry: %v", app.Status.ReconciledAt, statusRefreshTimeout)
 		//TODO: find existing Golang bug or create a new one
 		reconciledAtStr := "never"
 		if app.Status.ReconciledAt != nil {
 			reconciledAtStr = app.Status.ReconciledAt.String()
 		}
-		reason = fmt.Sprintf("comparison expired. reconciledAt: %v, expiry: %v", reconciledAtStr, statusRefreshTimeout)
+		reason = fmt.Sprintf("comparison expired, requesting refresh. reconciledAt: %v, expiry: %v", reconciledAtStr, statusRefreshTimeout)
+		if hardExpired {
+			reason = fmt.Sprintf("comparison expired, requesting hard refresh. reconciledAt: %v, expiry: %v", reconciledAtStr, statusHardRefreshTimeout)
+			refreshType = appv1.RefreshTypeHard
+		}
 	} else if !app.Spec.Destination.Equals(app.Status.Sync.ComparedTo.Destination) {
 		reason = "spec.destination differs"
 	} else if requested, level := ctrl.isRefreshRequested(app.Name); requested {
@@ -1660,6 +1668,10 @@ func (ctrl *ApplicationController) canProcessApp(obj interface{}) bool {
 }
 
 func (ctrl *ApplicationController) newApplicationInformerAndLister() (cache.SharedIndexInformer, applisters.ApplicationLister) {
+	refreshTimeout := ctrl.statusRefreshTimeout
+	if ctrl.statusHardRefreshTimeout.Seconds() != 0 && (ctrl.statusHardRefreshTimeout < ctrl.statusRefreshTimeout) {
+		refreshTimeout = ctrl.statusHardRefreshTimeout
+	}
 	informer := cache.NewSharedIndexInformer(
 		&cache.ListWatch{
 			ListFunc: func(options metav1.ListOptions) (apiruntime.Object, error) {
@@ -1670,7 +1682,7 @@ func (ctrl *ApplicationController) newApplicationInformerAndLister() (cache.Shar
 			},
 		},
 		&appv1.Application{},
-		ctrl.statusRefreshTimeout,
+		refreshTimeout,
 		cache.Indexers{
 			cache.NamespaceIndex: func(obj interface{}) ([]string, error) {
 				app, ok := obj.(*appv1.Application)

diff --git a/controller/appcontroller_test.go b/controller/appcontroller_test.go
@@ -103,6 +103,7 @@ func newFakeController(data *fakeData) *ApplicationController {
 		),
 		kubectl,
 		time.Minute,
+		time.Hour,
 		time.Minute,
 		common.DefaultPortArgoCDMetrics,
 		data.metricsCacheExpiration,
@@ -857,22 +858,22 @@ func TestNeedRefreshAppStatus(t *testing.T) {
 	}
 
 	// no need to refresh just reconciled application
-	needRefresh, _, _ := ctrl.needRefreshAppStatus(app, 1*time.Hour)
+	needRefresh, _, _ := ctrl.needRefreshAppStatus(app, 1*time.Hour, 2*time.Hour)
 	assert.False(t, needRefresh)
 
 	// refresh app using the 'deepest' requested comparison level
 	ctrl.requestAppRefresh(app.Name, CompareWithRecent.Pointer(), nil)
 	ctrl.requestAppRefresh(app.Name, ComparisonWithNothing.Pointer(), nil)
 
-	needRefresh, refreshType, compareWith := ctrl.needRefreshAppStatus(app, 1*time.Hour)
+	needRefresh, refreshType, compareWith := ctrl.needRefreshAppStatus(app, 1*time.Hour, 2*time.Hour)
 	assert.True(t, needRefresh)
 	assert.Equal(t, argoappv1.RefreshTypeNormal, refreshType)
 	assert.Equal(t, CompareWithRecent, compareWith)
 
 	// refresh application which status is not reconciled using latest commit
 	app.Status.Sync = argoappv1.SyncStatus{Status: argoappv1.SyncStatusCodeUnknown}
 
-	needRefresh, refreshType, compareWith = ctrl.needRefreshAppStatus(app, 1*time.Hour)
+	needRefresh, refreshType, compareWith = ctrl.needRefreshAppStatus(app, 1*time.Hour, 2*time.Hour)
 	assert.True(t, needRefresh)
 	assert.Equal(t, argoappv1.RefreshTypeNormal, refreshType)
 	assert.Equal(t, CompareWithLatestForceResolve, compareWith)
@@ -883,12 +884,31 @@ func TestNeedRefreshAppStatus(t *testing.T) {
 		ctrl.requestAppRefresh(app.Name, CompareWithRecent.Pointer(), nil)
 		reconciledAt := metav1.NewTime(time.Now().UTC().Add(-1 * time.Hour))
 		app.Status.ReconciledAt = &reconciledAt
-		needRefresh, refreshType, compareWith = ctrl.needRefreshAppStatus(app, 1*time.Minute)
+		needRefresh, refreshType, compareWith = ctrl.needRefreshAppStatus(app, 1*time.Minute, 2*time.Hour)
 		assert.True(t, needRefresh)
 		assert.Equal(t, argoappv1.RefreshTypeNormal, refreshType)
 		assert.Equal(t, CompareWithLatestForceResolve, compareWith)
 	}
 
+	{
+		// refresh app using the 'latest' level if comparison expired for hard refresh
+		app := app.DeepCopy()
+		app.Status.Sync = argoappv1.SyncStatus{
+			Status: argoappv1.SyncStatusCodeSynced,
+			ComparedTo: argoappv1.ComparedTo{
+				Source:      app.Spec.Source,
+				Destination: app.Spec.Destination,
+			},
+		}
+		ctrl.requestAppRefresh(app.Name, CompareWithRecent.Pointer(), nil)
+		reconciledAt := metav1.NewTime(time.Now().UTC().Add(-1 * time.Hour))
+		app.Status.ReconciledAt = &reconciledAt
+		needRefresh, refreshType, compareWith = ctrl.needRefreshAppStatus(app, 2*time.Hour, 1*time.Minute)
+		assert.True(t, needRefresh)
+		assert.Equal(t, argoappv1.RefreshTypeHard, refreshType)
+		assert.Equal(t, CompareWithLatest, compareWith)
+	}
+
 	{
 		app := app.DeepCopy()
 		// execute hard refresh if app has refresh annotation
@@ -897,7 +917,7 @@ func TestNeedRefreshAppStatus(t *testing.T) {
 		app.Annotations = map[string]string{
 			v1alpha1.AnnotationKeyRefresh: string(argoappv1.RefreshTypeHard),
 		}
-		needRefresh, refreshType, compareWith = ctrl.needRefreshAppStatus(app, 1*time.Hour)
+		needRefresh, refreshType, compareWith = ctrl.needRefreshAppStatus(app, 1*time.Hour, 2*time.Hour)
 		assert.True(t, needRefresh)
 		assert.Equal(t, argoappv1.RefreshTypeHard, refreshType)
 		assert.Equal(t, CompareWithLatestForceResolve, compareWith)
@@ -915,7 +935,7 @@ func TestNeedRefreshAppStatus(t *testing.T) {
 			}},
 		}
 
-		needRefresh, refreshType, compareWith = ctrl.needRefreshAppStatus(app, 1*time.Hour)
+		needRefresh, refreshType, compareWith = ctrl.needRefreshAppStatus(app, 1*time.Hour, 2*time.Hour)
 		assert.True(t, needRefresh)
 		assert.Equal(t, argoappv1.RefreshTypeNormal, refreshType)
 		assert.Equal(t, CompareWithLatestForceResolve, compareWith)

diff --git a/docs/operator-manual/server-commands/argocd-application-controller.md b/docs/operator-manual/server-commands/argocd-application-controller.md
@@ -13,6 +13,7 @@ argocd-application-controller [flags]
 ### Options
 
 ```
+      --app-hard-resync int                   Time period in seconds for application hard resync.
       --app-resync int                        Time period in seconds for application resync. (default 180)
       --app-state-cache-expiration duration   Cache expiration for app state (default 1h0m0s)
       --as string                             Username to impersonate for the operation

diff --git a/manifests/base/application-controller/argocd-application-controller-statefulset.yaml b/manifests/base/application-controller/argocd-application-controller-statefulset.yaml
@@ -29,6 +29,12 @@ spec:
               name: argocd-cm
               key: timeout.reconciliation
               optional: true
+        - name: ARGOCD_HARD_RECONCILIATION_TIMEOUT
+          valueFrom:
+            configMapKeyRef:
+              name: argocd-cm
+              key: timeout.hard.reconciliation
+              optional: true
         - name: ARGOCD_APPLICATION_CONTROLLER_REPO_SERVER
           valueFrom:
               configMapKeyRef:

diff --git a/manifests/core-install.yaml b/manifests/core-install.yaml
@@ -9331,6 +9331,12 @@ spec:
               key: timeout.reconciliation
               name: argocd-cm
               optional: true
+        - name: ARGOCD_HARD_RECONCILIATION_TIMEOUT
+          valueFrom:
+            configMapKeyRef:
+              key: timeout.hard.reconciliation
+              name: argocd-cm
+              optional: true
         - name: ARGOCD_APPLICATION_CONTROLLER_REPO_SERVER
           valueFrom:
             configMapKeyRef:

diff --git a/manifests/ha/install.yaml b/manifests/ha/install.yaml
@@ -10684,6 +10684,12 @@ spec:
               key: timeout.reconciliation
               name: argocd-cm
               optional: true
+        - name: ARGOCD_HARD_RECONCILIATION_TIMEOUT
+          valueFrom:
+            configMapKeyRef:
+              key: timeout.hard.reconciliation
+              name: argocd-cm
+              optional: true
         - name: ARGOCD_APPLICATION_CONTROLLER_REPO_SERVER
           valueFrom:
             configMapKeyRef:

diff --git a/manifests/ha/namespace-install.yaml b/manifests/ha/namespace-install.yaml
@@ -1971,6 +1971,12 @@ spec:
               key: timeout.reconciliation
               name: argocd-cm
               optional: true
+        - name: ARGOCD_HARD_RECONCILIATION_TIMEOUT
+          valueFrom:
+            configMapKeyRef:
+              key: timeout.hard.reconciliation
+              name: argocd-cm
+              optional: true
         - name: ARGOCD_APPLICATION_CONTROLLER_REPO_SERVER
           valueFrom:
             configMapKeyRef:

diff --git a/manifests/install.yaml b/manifests/install.yaml
@@ -10008,6 +10008,12 @@ spec:
               key: timeout.reconciliation
               name: argocd-cm
               optional: true
+        - name: ARGOCD_HARD_RECONCILIATION_TIMEOUT
+          valueFrom:
+            configMapKeyRef:
+              key: timeout.hard.reconciliation
+              name: argocd-cm
+              optional: true
         - name: ARGOCD_APPLICATION_CONTROLLER_REPO_SERVER
           valueFrom:
             configMapKeyRef:

diff --git a/manifests/namespace-install.yaml b/manifests/namespace-install.yaml
@@ -1295,6 +1295,12 @@ spec:
               key: timeout.reconciliation
               name: argocd-cm
               optional: true
+        - name: ARGOCD_HARD_RECONCILIATION_TIMEOUT
+          valueFrom:
+            configMapKeyRef:
+              key: timeout.hard.reconciliation
+              name: argocd-cm
+              optional: true
         - name: ARGOCD_APPLICATION_CONTROLLER_REPO_SERVER
           valueFrom:
             configMapKeyRef: