For Ansible/Helm-based operators, add Liveness and Readiness probe

operator-framework · Dec 16, 2020 · b7678e1 · b7678e1
1 parent 77f1b5e
commit b7678e1
Show file tree

Hide file tree

Showing 11 changed files with 148 additions and 11 deletions.
diff --git a/changelog/fragments/liveness_readiness_probe_for_operator.yaml b/changelog/fragments/liveness_readiness_probe_for_operator.yaml
@@ -0,0 +1,42 @@
+# entries is a list of entries to include in
+# release notes and/or the migration guide
+entries:
+  - description: >
+      For Ansible/Helm-based operators, add Liveness and Readiness probe.
+
+    # kind is one of:
+    # - addition
+    # - change
+    # - deprecation
+    # - removal
+    # - bugfix
+    kind: "addition"
+
+    # Is this a breaking change?
+    breaking: false
+
+    # Migration can be defined to automatically add a section to
+    # the migration guide. This is required for breaking changes.
+    migration:
+      header: (Optional) For Ansible/Helm-based operators, add Liveness and Readiness probe
+      body: >
+        New projects built with the tool will have the probes configured by default. The endpoints `/healthz` and
+        `readyz` are available now in the image based provided.
+
+        You can update your pre-existent project to use them. For that update the Docker file to use the latest
+        release based image and then, add in the `config/default/manager`:
+
+        ```yml
+          livenessProbe:
+            httpGet:
+              path: /healthz
+              port: 8081
+            initialDelaySeconds: 15
+            periodSeconds: 20
+          readinessProbe:
+            httpGet:
+              path: /readyz
+              port: 8081
+            initialDelaySeconds: 5
+            periodSeconds: 10
+        ```
diff --git a/internal/ansible/flags/flag.go b/internal/ansible/flags/flag.go
@@ -32,6 +32,7 @@ type Flags struct {
 	AnsibleRolesPath        string
 	AnsibleCollectionsPath  string
 	MetricsAddress          string
+	ProbeAddr               string
 	LeaderElectionID        string
 	LeaderElectionNamespace string
 	AnsibleArgs             string
@@ -82,6 +83,11 @@ func (f *Flags) AddTo(flagSet *pflag.FlagSet) {
 		":8080",
 		"The address the metric endpoint binds to",
 	)
+	flagSet.StringVar(&f.ProbeAddr,
+		"health-probe-bind-address",
+		":8081",
+		"The address the probe endpoint binds to.",
+	)
 	flagSet.BoolVar(&f.EnableLeaderElection,
 		"enable-leader-election",
 		false,

diff --git a/internal/cmd/ansible-operator/run/cmd.go b/internal/cmd/ansible-operator/run/cmd.go
@@ -45,11 +45,7 @@ import (
 	sdkVersion "github.com/operator-framework/operator-sdk/internal/version"
 )
 
-var (
-	metricsHost           = "0.0.0.0"
-	log                   = logf.Log.WithName("cmd")
-	healthProbePort int32 = 6789
-)
+var log = logf.Log.WithName("cmd")
 
 func printVersion() {
 	log.Info("Version",
@@ -105,8 +101,8 @@ func run(cmd *cobra.Command, f *flags.Flags) {
 	// Set default manager options
 	// TODO: probably should expose the host & port as an environment variables
 	options := manager.Options{
-		HealthProbeBindAddress:     fmt.Sprintf("%s:%d", metricsHost, healthProbePort),
 		MetricsBindAddress:         f.MetricsAddress,
+		HealthProbeBindAddress:     f.ProbeAddr,
 		LeaderElection:             f.EnableLeaderElection,
 		LeaderElectionID:           f.LeaderElectionID,
 		LeaderElectionResourceLock: resourcelock.ConfigMapsResourceLock,
@@ -148,6 +144,15 @@ func run(cmd *cobra.Command, f *flags.Flags) {
 		os.Exit(1)
 	}
 
+	if err := mgr.AddHealthzCheck("health", healthz.Ping); err != nil {
+		log.Error(err, "Unable to set up health check")
+		os.Exit(1)
+	}
+	if err := mgr.AddReadyzCheck("check", healthz.Ping); err != nil {
+		log.Error(err, "Unable to set up ready check")
+		os.Exit(1)
+	}
+
 	cMap := controllermap.NewControllerMap()
 	watches, err := watches.Load(f.WatchesFile, f.MaxConcurrentReconciles, f.AnsibleVerbosity)
 	if err != nil {
@@ -183,11 +188,6 @@ func run(cmd *cobra.Command, f *flags.Flags) {
 		}, w.Blacklist)
 	}
 
-	err = mgr.AddHealthzCheck("ping", healthz.Ping)
-	if err != nil {
-		log.Error(err, "Failed to add Healthz check.")
-	}
-
 	done := make(chan error)
 
 	// start the proxy

diff --git a/internal/cmd/helm-operator/run/cmd.go b/internal/cmd/helm-operator/run/cmd.go
@@ -26,6 +26,7 @@ import (
 	"k8s.io/client-go/tools/leaderelection/resourcelock"
 	"sigs.k8s.io/controller-runtime/pkg/cache"
 	"sigs.k8s.io/controller-runtime/pkg/client/config"
+	"sigs.k8s.io/controller-runtime/pkg/healthz"
 	logf "sigs.k8s.io/controller-runtime/pkg/log"
 	zapf "sigs.k8s.io/controller-runtime/pkg/log/zap"
 	"sigs.k8s.io/controller-runtime/pkg/manager"
@@ -96,6 +97,7 @@ func run(cmd *cobra.Command, f *flags.Flags) {
 	// Set default manager options
 	options := manager.Options{
 		MetricsBindAddress:         f.MetricsAddress,
+		HealthProbeBindAddress:     f.ProbeAddr,
 		LeaderElection:             f.EnableLeaderElection,
 		LeaderElectionID:           f.LeaderElectionID,
 		LeaderElectionResourceLock: resourcelock.ConfigMapsResourceLock,
@@ -130,6 +132,15 @@ func run(cmd *cobra.Command, f *flags.Flags) {
 		os.Exit(1)
 	}
 
+	if err := mgr.AddHealthzCheck("health", healthz.Ping); err != nil {
+		log.Error(err, "Unable to set up health check")
+		os.Exit(1)
+	}
+	if err := mgr.AddReadyzCheck("check", healthz.Ping); err != nil {
+		log.Error(err, "unable to set up ready check")
+		os.Exit(1)
+	}
+
 	ws, err := watches.Load(f.WatchesFile)
 	if err != nil {
 		log.Error(err, "Failed to create new manager factories.")

diff --git a/internal/helm/flags/flag.go b/internal/helm/flags/flag.go
@@ -30,6 +30,7 @@ type Flags struct {
 	LeaderElectionID        string
 	LeaderElectionNamespace string
 	MaxConcurrentReconciles int
+	ProbeAddr               string
 }
 
 // AddTo - Add the helm operator flags to the the flagset
@@ -49,6 +50,11 @@ func (f *Flags) AddTo(flagSet *pflag.FlagSet) {
 		":8080",
 		"The address the metric endpoint binds to",
 	)
+	flagSet.StringVar(&f.ProbeAddr,
+		"health-probe-bind-address",
+		":8081",
+		"The address the probe endpoint binds to.",
+	)
 	flagSet.BoolVar(&f.EnableLeaderElection,
 		"enable-leader-election",
 		false,

diff --git a/internal/plugins/ansible/v1/scaffolds/internal/templates/config/manager/manager.go b/internal/plugins/ansible/v1/scaffolds/internal/templates/config/manager/manager.go
@@ -78,5 +78,17 @@ spec:
             - name: ANSIBLE_GATHERING
               value: explicit
           image: {{ .Image }}
+          livenessProbe:
+            httpGet:
+              path: /readyz
+              port: 8081
+            initialDelaySeconds: 15
+            periodSeconds: 20
+          readinessProbe:
+            httpGet:
+              path: /healthz
+              port: 8081
+            initialDelaySeconds: 5
+            periodSeconds: 10		   	
       terminationGracePeriodSeconds: 10
 `
diff --git a/internal/plugins/helm/v1/scaffolds/internal/templates/config/manager/manager.go b/internal/plugins/helm/v1/scaffolds/internal/templates/config/manager/manager.go
@@ -75,6 +75,18 @@ spec:
         - "--enable-leader-election"
         - "--leader-election-id={{ .ProjectName }}"
         name: manager
+        livenessProbe:
+          httpGet:
+            path: /readyz
+            port: 8081
+          initialDelaySeconds: 15
+          periodSeconds: 20
+        readinessProbe:
+          httpGet:
+            path: /healthz
+            port: 8081
+          initialDelaySeconds: 5
+          periodSeconds: 10
         resources:
           limits:
             cpu: 100m

diff --git a/...ansible/memcached-operator/bundle/manifests/memcached-operator.clusterserviceversion.yaml b/...ansible/memcached-operator/bundle/manifests/memcached-operator.clusterserviceversion.yaml
@@ -124,7 +124,19 @@ spec:
                 - name: ANSIBLE_GATHERING
                   value: explicit
                 image: quay.io/example/memcached-operator:v0.0.1
+                livenessProbe:
+                  httpGet:
+                    path: /readyz
+                    port: 8081
+                  initialDelaySeconds: 15
+                  periodSeconds: 20
                 name: manager
+                readinessProbe:
+                  httpGet:
+                    path: /healthz
+                    port: 8081
+                  initialDelaySeconds: 5
+                  periodSeconds: 10
                 resources: {}
               terminationGracePeriodSeconds: 10
       permissions:

diff --git a/testdata/ansible/memcached-operator/config/manager/manager.yaml b/testdata/ansible/memcached-operator/config/manager/manager.yaml
@@ -31,4 +31,16 @@ spec:
             - name: ANSIBLE_GATHERING
               value: explicit
           image: controller:latest
+          livenessProbe:
+            httpGet:
+              path: /readyz
+              port: 8081
+            initialDelaySeconds: 15
+            periodSeconds: 20
+          readinessProbe:
+            httpGet:
+              path: /healthz
+              port: 8081
+            initialDelaySeconds: 5
+            periodSeconds: 10		   	
       terminationGracePeriodSeconds: 10
diff --git a/...ta/helm/memcached-operator/bundle/manifests/memcached-operator.clusterserviceversion.yaml b/...ta/helm/memcached-operator/bundle/manifests/memcached-operator.clusterserviceversion.yaml
@@ -209,7 +209,19 @@ spec:
                 - --enable-leader-election
                 - --leader-election-id=memcached-operator
                 image: quay.io/example/memcached-operator:v0.0.1
+                livenessProbe:
+                  httpGet:
+                    path: /readyz
+                    port: 8081
+                  initialDelaySeconds: 15
+                  periodSeconds: 20
                 name: manager
+                readinessProbe:
+                  httpGet:
+                    path: /healthz
+                    port: 8081
+                  initialDelaySeconds: 5
+                  periodSeconds: 10
                 resources:
                   limits:
                     cpu: 100m

diff --git a/testdata/helm/memcached-operator/config/manager/manager.yaml b/testdata/helm/memcached-operator/config/manager/manager.yaml
@@ -28,6 +28,18 @@ spec:
         - "--enable-leader-election"
         - "--leader-election-id=memcached-operator"
         name: manager
+        livenessProbe:
+          httpGet:
+            path: /readyz
+            port: 8081
+          initialDelaySeconds: 15
+          periodSeconds: 20
+        readinessProbe:
+          httpGet:
+            path: /healthz
+            port: 8081
+          initialDelaySeconds: 5
+          periodSeconds: 10
         resources:
           limits:
             cpu: 100m