Add probes to deployment and probe server to ctrlr

chart/orkestra/templates/deployment.yaml

@@ -52,6 +52,20 @@ spec:
             value: {{ include "orkestra.serviceAccountName" . }} 
           resources:
             {{- toYaml .Values.resources | nindent 12 }}
+          # define a liveness probe that checks every 5 seconds, starting after 5 seconds
+          livenessProbe:
+            httpGet:
+              path: /live
+              port: 8086
+            initialDelaySeconds: 5
+            periodSeconds: 5
+
+          # define a readiness probe that checks every 5 seconds
+          readinessProbe:
+            httpGet:
+              path: /ready
+              port: 8086
+            periodSeconds: 5
       {{- with .Values.nodeSelector }}
       nodeSelector:
         {{- toYaml . | nindent 8 }}

go.mod

@@ -9,6 +9,7 @@ require (
 	github.com/go-logr/logr v0.3.0
 	github.com/gofrs/flock v0.8.0
 	github.com/google/go-cmp v0.5.2
+	github.com/heptiolabs/healthcheck v0.0.0-20180807145615-6ff867650f40 // indirect
 	github.com/jinzhu/copier v0.2.8
 	github.com/mitchellh/hashstructure/v2 v2.0.1
 	go.opencensus.io v0.22.5 // indirect

go.sum

@@ -551,6 +551,8 @@ github.com/hashicorp/vault/sdk v0.1.13/go.mod h1:B+hVj7TpuQY1Y/GPbCpffmgd+tSEwvh
 github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM=
 github.com/hashicorp/yamux v0.0.0-20181012175058-2f1d1f20f75d/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM=
 github.com/helm/helm-2to3 v0.5.1/go.mod h1:AXFpQX2cSQpss+47ROPEeu7Sm4+CRJ1jKWCEQdHP3/c=
+github.com/heptiolabs/healthcheck v0.0.0-20180807145615-6ff867650f40 h1:GT4RsKmHh1uZyhmTkWJTDALRjSHYQp6FRKrotf0zhAs=
+github.com/heptiolabs/healthcheck v0.0.0-20180807145615-6ff867650f40/go.mod h1:NtmN9h8vrTveVQRLHcX2HQ5wIPBDCsZ351TGbZWgg38=
 github.com/howeyc/gopass v0.0.0-20170109162249-bf9dde6d0d2c/go.mod h1:lADxMC39cJJqL93Duh1xhAs4I2Zs8mKS89XWXFGp9cs=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/huandu/xstrings v1.3.1 h1:4jgBlKK6tLKFvO8u5pmYjG91cqytmDCDvGh7ECVFfFs=

main.go

@@ -7,6 +7,7 @@ import (
 	"flag"
 	"os"
 
+	"github.com/Azure/Orkestra/pkg"
 	"github.com/Azure/Orkestra/pkg/registry"
 	"github.com/Azure/Orkestra/pkg/workflow"
 	"k8s.io/apimachinery/pkg/runtime"
@@ -133,6 +134,14 @@ func main() {
 	}
 	// +kubebuilder:scaffold:builder
 
+	probe, err := pkg.ProbeHandler(stagingRepoURL, "health")
+	if err != nil {
+		setupLog.Error(err, "unable to start readiness/liveness probes", "controller", "ApplicationGroup")
+		os.Exit(1)
+	}
+
+	probe.Start("8086")
+
 	setupLog.Info("starting manager")
 	if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil {
 		setupLog.Error(err, "problem running manager")

pkg/probe.go

@@ -0,0 +1,29 @@
+package pkg
+
+import (
+	"net"
+	"net/http"
+	"time"
+
+	"github.com/heptiolabs/healthcheck"
+)
+
+type Probe struct {
+	health healthcheck.Handler
+}
+
+func ProbeHandler(chartmuseumAddress, endpoint string) (*Probe, error) {
+	health := healthcheck.NewHandler()
+	// Liveness check verifies that the number of goroutines are below threshold
+	health.AddLivenessCheck("goroutine-threshold", healthcheck.GoroutineCountCheck(100))
+	// Readiness check verifies that chartmuseum is up and serving traffic
+	health.AddReadinessCheck("chartmuseum-ready", healthcheck.HTTPGetCheck(chartmuseumAddress+"/"+endpoint, time.Minute))
+
+	return &Probe{
+		health: health,
+	}, nil
+}
+
+func (p *Probe) Start(port string) {
+	go http.ListenAndServe(net.JoinHostPort("0.0.0.0", port), p.health) //nolint:errcheck
+}