chore: add start up probe for jetstream eventbus (#1735)

* chore: add start up probe for jetstream eventbus Signed-off-by: Derek Wang <whynowy@gmail.com> * . Signed-off-by: Derek Wang <whynowy@gmail.com> * . Signed-off-by: Derek Wang <whynowy@gmail.com>
argoproj · Mar 17, 2022 · f7bd6a0 · f7bd6a0
1 parent 4e85153
commit f7bd6a0
Show file tree

Hide file tree

Showing 16 changed files with 282 additions and 143 deletions.
diff --git a/api/event-bus.html b/api/event-bus.html
diff --git a/api/event-bus.md b/api/event-bus.md
diff --git a/api/jsonschema/schema.json b/api/jsonschema/schema.json
@@ -457,6 +457,13 @@
           "description": "JetStream configuration, if not specified, global settings in controller-config will be used. See https://docs.nats.io/running-a-nats-service/configuration#jetstream. Only configure \"max_memory_store\" or \"max_file_store\", do not set \"store_dir\" as it has been hardcoded.",
           "type": "string"
         },
+        "startArgs": {
+          "description": "Optional arguments to start nats-server. For example, \"-D\" to enable debugging output, \"-DV\" to enable debugging and tracing. Check https://docs.nats.io/ for all the available arguments.",
+          "items": {
+            "type": "string"
+          },
+          "type": "array"
+        },
         "tolerations": {
           "description": "If specified, the pod's tolerations.",
           "items": {

diff --git a/api/openapi-spec/swagger.json b/api/openapi-spec/swagger.json
diff --git a/controllers/eventbus/controller.go b/controllers/eventbus/controller.go
@@ -84,8 +84,10 @@ func (r *reconciler) reconcile(ctx context.Context, eventBus *v1alpha1.EventBus)
 	eventBus.Status.InitConditions()
 	if err := ValidateEventBus(eventBus); err != nil {
 		log.Errorw("validation failed", zap.Error(err))
-		eventBus.Status.MarkDeployFailed("InvalidSpec", err.Error())
+		eventBus.Status.MarkNotConfigured("InvalidSpec", err.Error())
 		return err
+	} else {
+		eventBus.Status.MarkConfigured()
 	}
 	return installer.Install(ctx, eventBus, r.client, r.config, log)
 }

diff --git a/controllers/eventbus/installer/exotic_nats.go b/controllers/eventbus/installer/exotic_nats.go
@@ -30,7 +30,6 @@ func (i *exoticNATSInstaller) Install(ctx context.Context) (*v1alpha1.BusConfig,
 		return nil, errors.New("invalid request")
 	}
 	i.eventBus.Status.MarkDeployed("Skipped", "Skip deployment because of using exotic config.")
-	i.eventBus.Status.MarkConfigured()
 	i.logger.Info("use exotic config")
 	busConfig := &v1alpha1.BusConfig{
 		NATS: natsObj.Exotic,

diff --git a/controllers/eventbus/installer/jetstream.go b/controllers/eventbus/installer/jetstream.go
@@ -81,7 +81,7 @@ func (r *jetStreamInstaller) Install(ctx context.Context) (*v1alpha1.BusConfig,
 	r.eventBus.Status.MarkDeployed("Succeeded", "JetStream is deployed")
 	return &v1alpha1.BusConfig{
 		JetStream: &v1alpha1.JetStreamConfig{
-			URL: fmt.Sprintf("nats://%s:%s", generateJetStreamServiceName(r.eventBus), strconv.Itoa(int(jsClientPort))),
+			URL: fmt.Sprintf("nats://%s.%s.svc.cluster.local:%s", generateJetStreamServiceName(r.eventBus), r.eventBus.Namespace, strconv.Itoa(int(jsClientPort))),
 			Auth: &v1alpha1.JetStreamAuth{
 				Token: &corev1.SecretKeySelector{
 					LocalObjectReference: corev1.LocalObjectReference{
@@ -103,9 +103,10 @@ func (r *jetStreamInstaller) buildJetStreamServiceSpec() corev1.ServiceSpec {
 			{Name: "metrics", Port: jsMetricsPort},
 			{Name: "monitor", Port: jsMonitorPort},
 		},
-		Type:      corev1.ServiceTypeClusterIP,
-		ClusterIP: corev1.ClusterIPNone,
-		Selector:  r.labels,
+		Type:                     corev1.ServiceTypeClusterIP,
+		ClusterIP:                corev1.ClusterIPNone,
+		PublishNotReadyAddresses: true,
+		Selector:                 r.labels,
 	}
 }
 
@@ -293,17 +294,29 @@ func (r *jetStreamInstaller) buildStatefulSetSpec(jsVersion *controllers.JetStre
 							{Name: "monitor", ContainerPort: jsMonitorPort},
 						},
 						Command: []string{jsVersion.StartCommand, "--config", "/etc/nats-config/nats-js.conf"},
+						Args:    js.StartArgs,
 						Env: []corev1.EnvVar{
 							{Name: "POD_NAME", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.name"}}},
 							{Name: "SERVER_NAME", Value: "$(POD_NAME)"},
 							{Name: "POD_NAMESPACE", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.namespace"}}},
-							{Name: "CLUSTER_ADVERTISE", Value: "$(POD_NAME)." + generateJetStreamServiceName(r.eventBus) + ".$(POD_NAMESPACE).svc"},
+							{Name: "CLUSTER_ADVERTISE", Value: "$(POD_NAME)." + generateJetStreamServiceName(r.eventBus) + ".$(POD_NAMESPACE).svc.cluster.local"},
 						},
 						VolumeMounts: []corev1.VolumeMount{
 							{Name: "config-volume", MountPath: "/etc/nats-config"},
 							{Name: "pid", MountPath: "/var/run/nats"},
 						},
 						SecurityContext: jsContainerSecurityContext,
+						StartupProbe: &corev1.Probe{
+							ProbeHandler: corev1.ProbeHandler{
+								HTTPGet: &corev1.HTTPGetAction{
+									Path: "/healthz",
+									Port: intstr.FromInt(int(jsMonitorPort)),
+								},
+							},
+							FailureThreshold:    30,
+							InitialDelaySeconds: 10,
+							TimeoutSeconds:      5,
+						},
 						LivenessProbe: &corev1.Probe{
 							ProbeHandler: corev1.ProbeHandler{
 								HTTPGet: &corev1.HTTPGetAction{
@@ -312,6 +325,7 @@ func (r *jetStreamInstaller) buildStatefulSetSpec(jsVersion *controllers.JetStre
 								},
 							},
 							InitialDelaySeconds: 10,
+							PeriodSeconds:       30,
 							TimeoutSeconds:      5,
 						},
 						Lifecycle: &corev1.Lifecycle{
@@ -340,7 +354,7 @@ func (r *jetStreamInstaller) buildStatefulSetSpec(jsVersion *controllers.JetStre
 						Ports: []corev1.ContainerPort{
 							{Name: "metrics", ContainerPort: jsMetricsPort},
 						},
-						Args:            []string{"-connz", "-routez", "-subz", "-varz", "-channelz", "-serverz", fmt.Sprintf("http://localhost:%s", strconv.Itoa(int(jsMonitorPort)))},
+						Args:            []string{"-connz", "-routez", "-subz", "-varz", "-prefix=nats", "-use_internal_server_id", "-jsz=all", fmt.Sprintf("http://localhost:%s", strconv.Itoa(int(jsMonitorPort)))},
 						SecurityContext: metricsContainerSecurityContext,
 					},
 				},
@@ -509,7 +523,7 @@ func (r *jetStreamInstaller) createConfigMap(ctx context.Context) error {
 	}
 	routes := []string{}
 	for j := 0; j < replicas; j++ {
-		routes = append(routes, fmt.Sprintf("nats://%s-%s.%s.%s.svc:%s", ssName, strconv.Itoa(j), svcName, r.eventBus.Namespace, strconv.Itoa(int(jsClusterPort))))
+		routes = append(routes, fmt.Sprintf("nats://%s-%s.%s.%s.svc.cluster.local:%s", ssName, strconv.Itoa(j), svcName, r.eventBus.Namespace, strconv.Itoa(int(jsClusterPort))))
 	}
 	settings := r.config.EventBus.JetStream.Settings
 	if x := r.eventBus.Spec.JetStream.Settings; x != nil {

diff --git a/controllers/eventbus/installer/nats.go b/controllers/eventbus/installer/nats.go
@@ -93,7 +93,6 @@ func (i *natsInstaller) Install(ctx context.Context) (*v1alpha1.BusConfig, error
 		return nil, err
 	}
 	i.eventBus.Status.MarkDeployed("Succeeded", "NATS is deployed")
-	i.eventBus.Status.MarkConfigured()
 	clusterID := generateClusterID(i.eventBus)
 	busConfig := &v1alpha1.BusConfig{
 		NATS: &v1alpha1.NATSConfig{

diff --git a/manifests/base/eventbus-controller/controller-config.yaml b/manifests/base/eventbus-controller/controller-config.yaml
@@ -20,13 +20,14 @@ data:
           # e.g. 20G. -1 means no limit, Up to 1TB if available
           max_file_store: 1TB
         versions:
-        - version: 2.7.3
-          natsImage: nats:2.7.3
-          configReloaderImage: natsio/nats-server-config-reloader:0.6.3
+        - version: 2.7.4
+          natsImage: nats:2.7.4
           metricsExporterImage: natsio/prometheus-nats-exporter:0.9.1
-          startCommand: /nats-server
-        - version: 2.7.3-alpine
-          natsImage: nats:2.7.3-alpine
           configReloaderImage: natsio/nats-server-config-reloader:0.6.3
+          startCommand: /nats-server
+        - version: 2.7.4-alpine
+          natsImage: nats:2.7.4-alpine
           metricsExporterImage: natsio/prometheus-nats-exporter:0.9.1
+          configReloaderImage: natsio/nats-server-config-reloader:0.6.3
           startCommand: nats-server
+
diff --git a/manifests/install.yaml b/manifests/install.yaml
@@ -278,7 +278,7 @@ subjects:
 ---
 apiVersion: v1
 data:
-  controller-config.yaml: |
+  controller-config.yaml: |+
     eventBus:
       nats:
         versions:
@@ -295,16 +295,17 @@ data:
           # e.g. 20G. -1 means no limit, Up to 1TB if available
           max_file_store: 1TB
         versions:
-        - version: 2.7.3
-          natsImage: nats:2.7.3
-          configReloaderImage: natsio/nats-server-config-reloader:0.6.3
+        - version: 2.7.4
+          natsImage: nats:2.7.4
           metricsExporterImage: natsio/prometheus-nats-exporter:0.9.1
-          startCommand: /nats-server
-        - version: 2.7.3-alpine
-          natsImage: nats:2.7.3-alpine
           configReloaderImage: natsio/nats-server-config-reloader:0.6.3
+          startCommand: /nats-server
+        - version: 2.7.4-alpine
+          natsImage: nats:2.7.4-alpine
           metricsExporterImage: natsio/prometheus-nats-exporter:0.9.1
+          configReloaderImage: natsio/nats-server-config-reloader:0.6.3
           startCommand: nats-server
+
 kind: ConfigMap
 metadata:
   name: argo-events-controller-config

diff --git a/manifests/namespace-install.yaml b/manifests/namespace-install.yaml
@@ -198,7 +198,7 @@ subjects:
 ---
 apiVersion: v1
 data:
-  controller-config.yaml: |
+  controller-config.yaml: |+
     eventBus:
       nats:
         versions:
@@ -215,16 +215,17 @@ data:
           # e.g. 20G. -1 means no limit, Up to 1TB if available
           max_file_store: 1TB
         versions:
-        - version: 2.7.3
-          natsImage: nats:2.7.3
-          configReloaderImage: natsio/nats-server-config-reloader:0.6.3
+        - version: 2.7.4
+          natsImage: nats:2.7.4
           metricsExporterImage: natsio/prometheus-nats-exporter:0.9.1
-          startCommand: /nats-server
-        - version: 2.7.3-alpine
-          natsImage: nats:2.7.3-alpine
           configReloaderImage: natsio/nats-server-config-reloader:0.6.3
+          startCommand: /nats-server
+        - version: 2.7.4-alpine
+          natsImage: nats:2.7.4-alpine
           metricsExporterImage: natsio/prometheus-nats-exporter:0.9.1
+          configReloaderImage: natsio/nats-server-config-reloader:0.6.3
           startCommand: nats-server
+
 kind: ConfigMap
 metadata:
   name: argo-events-controller-config