Add health check to the controller deployment (#1785)

controller/appcontroller.go

@@ -113,7 +113,10 @@ func NewApplicationController(
 	appInformer, appLister := ctrl.newApplicationInformerAndLister()
 	projInformer := v1alpha1.NewAppProjectInformer(applicationClientset, namespace, appResyncPeriod, cache.Indexers{})
 	metricsAddr := fmt.Sprintf("0.0.0.0:%d", metricsPort)
-	ctrl.metricsServer = metrics.NewMetricsServer(metricsAddr, appLister)
+	ctrl.metricsServer = metrics.NewMetricsServer(metricsAddr, appLister, func() error {
+		_, err := kubeClientset.Discovery().ServerVersion()
+		return err
+	})
 	stateCache := statecache.NewLiveStateCache(db, appInformer, ctrl.settings, kubectlCmd, ctrl.metricsServer, ctrl.handleAppUpdated)
 	appStateManager := NewAppStateManager(db, applicationClientset, repoClientset, namespace, kubectlCmd, ctrl.settings, stateCache, projInformer, ctrl.metricsServer)
 	ctrl.appInformer = appInformer

controller/metrics/metrics.go

@@ -13,6 +13,7 @@ import (
 	argoappv1 "github.com/argoproj/argo-cd/pkg/apis/application/v1alpha1"
 	applister "github.com/argoproj/argo-cd/pkg/client/listers/application/v1alpha1"
 	"github.com/argoproj/argo-cd/util/git"
+	"github.com/argoproj/argo-cd/util/healthz"
 )
 
 type MetricsServer struct {
@@ -59,12 +60,13 @@ var (
 )
 
 // NewMetricsServer returns a new prometheus server which collects application metrics
-func NewMetricsServer(addr string, appLister applister.ApplicationLister) *MetricsServer {
+func NewMetricsServer(addr string, appLister applister.ApplicationLister, healthCheck func() error) *MetricsServer {
 	mux := http.NewServeMux()
 	appRegistry := NewAppRegistry(appLister)
 	appRegistry.MustRegister(prometheus.NewProcessCollector(prometheus.ProcessCollectorOpts{}))
 	appRegistry.MustRegister(prometheus.NewGoCollector())
 	mux.Handle(MetricsPath, promhttp.HandlerFor(appRegistry, promhttp.HandlerOpts{}))
+	healthz.ServeHealthCheck(mux, healthCheck)
 
 	syncCounter := prometheus.NewCounterVec(
 		prometheus.CounterOpts{

controller/metrics/metrics_test.go

@@ -104,6 +104,10 @@ argocd_app_sync_status{name="my-app",namespace="argocd",project="default",sync_s
 argocd_app_sync_status{name="my-app",namespace="argocd",project="default",sync_status="Unknown"} 0
 `
 
+var noOpHealthCheck = func() error {
+	return nil
+}
+
 func newFakeApp(fakeApp string) *argoappv1.Application {
 	var app argoappv1.Application
 	err := yaml.Unmarshal([]byte(fakeApp), &app)
@@ -133,7 +137,7 @@ func newFakeLister(fakeApp ...string) (context.CancelFunc, applister.Application
 func testApp(t *testing.T, fakeApp string, expectedResponse string) {
 	cancel, appLister := newFakeLister(fakeApp)
 	defer cancel()
-	metricsServ := NewMetricsServer("localhost:8082", appLister)
+	metricsServ := NewMetricsServer("localhost:8082", appLister, noOpHealthCheck)
 	req, err := http.NewRequest("GET", "/metrics", nil)
 	assert.NoError(t, err)
 	rr := httptest.NewRecorder()
@@ -176,7 +180,7 @@ argocd_app_sync_total{name="my-app",namespace="argocd",phase="Succeeded",project
 func TestMetricsSyncCounter(t *testing.T) {
 	cancel, appLister := newFakeLister()
 	defer cancel()
-	metricsServ := NewMetricsServer("localhost:8082", appLister)
+	metricsServ := NewMetricsServer("localhost:8082", appLister, noOpHealthCheck)
 
 	fakeApp := newFakeApp(fakeApp)
 	metricsServ.IncSync(fakeApp, &argoappv1.OperationState{Phase: argoappv1.OperationRunning})
@@ -217,7 +221,7 @@ argocd_app_reconcile_count{name="my-app",namespace="argocd",project="important-p
 func TestReconcileMetrics(t *testing.T) {
 	cancel, appLister := newFakeLister()
 	defer cancel()
-	metricsServ := NewMetricsServer("localhost:8082", appLister)
+	metricsServ := NewMetricsServer("localhost:8082", appLister, noOpHealthCheck)
 
 	fakeApp := newFakeApp(fakeApp)
 	metricsServ.IncReconcile(fakeApp, 5*time.Second)

manifests/base/application-controller/argocd-application-controller-deployment.yaml

@@ -30,7 +30,14 @@ spec:
         ports:
         - containerPort: 8082
         readinessProbe:
-          tcpSocket:
+          httpGet:
+            path: /healthz
+            port: 8082
+          initialDelaySeconds: 5
+          periodSeconds: 10
+        livenessProbe:
+          httpGet:
+            path: /healthz
             port: 8082
           initialDelaySeconds: 5
           periodSeconds: 10

manifests/ha/install.yaml

@@ -2207,14 +2207,21 @@ spec:
         - argocd
         image: argoproj/argocd:latest
         imagePullPolicy: Always
+        livenessProbe:
+          httpGet:
+            path: /healthz
+            port: 8082
+          initialDelaySeconds: 5
+          periodSeconds: 10
         name: argocd-application-controller
         ports:
         - containerPort: 8082
         readinessProbe:
+          httpGet:
+            path: /healthz
+            port: 8082
           initialDelaySeconds: 5
           periodSeconds: 10
-          tcpSocket:
-            port: 8082
       serviceAccountName: argocd-application-controller
 ---
 apiVersion: apps/v1

manifests/ha/namespace-install.yaml

@@ -2122,14 +2122,21 @@ spec:
         - argocd
         image: argoproj/argocd:latest
         imagePullPolicy: Always
+        livenessProbe:
+          httpGet:
+            path: /healthz
+            port: 8082
+          initialDelaySeconds: 5
+          periodSeconds: 10
         name: argocd-application-controller
         ports:
         - containerPort: 8082
         readinessProbe:
+          httpGet:
+            path: /healthz
+            port: 8082
           initialDelaySeconds: 5
           periodSeconds: 10
-          tcpSocket:
-            port: 8082
       serviceAccountName: argocd-application-controller
 ---
 apiVersion: apps/v1

manifests/install.yaml

@@ -1971,14 +1971,21 @@ spec:
         - "10"
         image: argoproj/argocd:latest
         imagePullPolicy: Always
+        livenessProbe:
+          httpGet:
+            path: /healthz
+            port: 8082
+          initialDelaySeconds: 5
+          periodSeconds: 10
         name: argocd-application-controller
         ports:
         - containerPort: 8082
         readinessProbe:
+          httpGet:
+            path: /healthz
+            port: 8082
           initialDelaySeconds: 5
           periodSeconds: 10
-          tcpSocket:
-            port: 8082
       serviceAccountName: argocd-application-controller
 ---
 apiVersion: apps/v1

manifests/namespace-install.yaml

@@ -1886,14 +1886,21 @@ spec:
         - "10"
         image: argoproj/argocd:latest
         imagePullPolicy: Always
+        livenessProbe:
+          httpGet:
+            path: /healthz
+            port: 8082
+          initialDelaySeconds: 5
+          periodSeconds: 10
         name: argocd-application-controller
         ports:
         - containerPort: 8082
         readinessProbe:
+          httpGet:
+            path: /healthz
+            port: 8082
           initialDelaySeconds: 5
           periodSeconds: 10
-          tcpSocket:
-            port: 8082
       serviceAccountName: argocd-application-controller
 ---
 apiVersion: apps/v1