Implement gRPC Health Checking Protocol + add readiness/liveness prob…

…es to vizier-core (kubeflow#270)

* Ensure vizier-core never been stuck too long waiting for DB conn

Signed-off-by: Koichiro Den <den@valinux.co.jp>

* Add standard Health gRPC service

Signed-off-by: Koichiro Den <den@valinux.co.jp>

* Change db.New to return error instead of exit(1) with log.Fatal

Signed-off-by: Koichiro Den <den@valinux.co.jp>

* Add SelectOne() to VizierDBInterface

Signed-off-by: Koichiro Den <den@valinux.co.jp>

* Rename import for later convenience

Signed-off-by: Koichiro Den <den@valinux.co.jp>

* Implement and register Health Server for Katib manager

Signed-off-by: Koichiro Den <den@valinux.co.jp>

* Add readiness/liveness probes to vizier-core

Signed-off-by: Koichiro Den <den@valinux.co.jp>

* Update test codebase

Fixes: 61ac5607353 ("Add SelectOne() to VizierDBInterface")
Signed-off-by: Koichiro Den <den@valinux.co.jp>

cmd/manager/Dockerfile

@@ -6,6 +6,9 @@ RUN go build -o vizier-manager
 
 FROM alpine:3.7
 WORKDIR /app
+RUN GRPC_HEALTH_PROBE_VERSION=v0.2.0 && \
+    wget -qO/bin/grpc_health_probe https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/${GRPC_HEALTH_PROBE_VERSION}/grpc_health_probe-linux-amd64 && \
+    chmod +x /bin/grpc_health_probe
 COPY --from=build-env /go/src/github.com/kubeflow/katib/cmd/manager/vizier-manager /app/
 COPY --from=build-env /go/src/github.com/kubeflow/katib/pkg/manager/visualise /
 ENTRYPOINT ["./vizier-manager"]

manifests/vizier/core/deployment.yaml

@@ -30,6 +30,14 @@ spec:
         ports:
         - name: api
           containerPort: 6789
+        readinessProbe:
+          exec:
+            command: ["/bin/grpc_health_probe", "-addr=:6789"]
+          initialDelaySeconds: 5
+        livenessProbe:
+          exec:
+            command: ["/bin/grpc_health_probe", "-addr=:6789"]
+          initialDelaySeconds: 10
 #        resources:
 #          requests:
 #            cpu: 500m

pkg/api/Makefile

@@ -1,2 +1,4 @@
 api.pb.go: api.proto
 	protoc -I. api.proto --go_out=plugins=grpc:.
+health.pb.go: health/health.proto
+	protoc -I. health/health.proto --go_out=plugins=grpc:.

pkg/api/build.sh

@@ -1,4 +1,6 @@
-docker run -it --rm -v $PWD:$(pwd) -w $(pwd) znly/protoc --python_out=plugins=grpc:./python --go_out=plugins=grpc:. -I. api.proto
-docker run -it --rm -v $PWD:$(pwd) -w $(pwd) znly/protoc --plugin=protoc-gen-grpc=/usr/bin/grpc_python_plugin --python_out=./python --grpc_out=./python -I. api.proto
-docker run -it --rm -v $PWD:$(pwd) -w $(pwd) znly/protoc --grpc-gateway_out=logtostderr=true:. -I. api.proto
-docker run -it --rm -v $PWD:$(pwd) -w $(pwd) znly/protoc --swagger_out=logtostderr=true:. -I. api.proto
+for proto in api.proto health/health.proto; do
+  docker run -it --rm -v $PWD:$(pwd) -w $(pwd) znly/protoc --python_out=plugins=grpc:./python --go_out=plugins=grpc:. -I. $proto
+  docker run -it --rm -v $PWD:$(pwd) -w $(pwd) znly/protoc --plugin=protoc-gen-grpc=/usr/bin/grpc_python_plugin --python_out=./python --grpc_out=./python -I. $proto
+  docker run -it --rm -v $PWD:$(pwd) -w $(pwd) znly/protoc --grpc-gateway_out=logtostderr=true:. -I. $proto
+  docker run -it --rm -v $PWD:$(pwd) -w $(pwd) znly/protoc --swagger_out=logtostderr=true:. -I. $proto
+done

pkg/api/health/health.proto

@@ -0,0 +1,20 @@
+syntax = "proto3";
+
+package grpc.health.v1;
+
+service Health {
+    rpc Check(HealthCheckRequest) returns (HealthCheckResponse);
+}
+
+message HealthCheckRequest {
+  string service = 1;
+}
+
+message HealthCheckResponse {
+  enum ServingStatus {
+    UNKNOWN = 0;
+    SERVING = 1;
+    NOT_SERVING = 2;
+  }
+  ServingStatus status = 1;
+}

pkg/api/health/health.swagger.json

@@ -0,0 +1,37 @@
+{
+  "swagger": "2.0",
+  "info": {
+    "title": "health/health.proto",
+    "version": "version not set"
+  },
+  "schemes": [
+    "http",
+    "https"
+  ],
+  "consumes": [
+    "application/json"
+  ],
+  "produces": [
+    "application/json"
+  ],
+  "paths": {},
+  "definitions": {
+    "HealthCheckResponseServingStatus": {
+      "type": "string",
+      "enum": [
+        "UNKNOWN",
+        "SERVING",
+        "NOT_SERVING"
+      ],
+      "default": "UNKNOWN"
+    },
+    "v1HealthCheckResponse": {
+      "type": "object",
+      "properties": {
+        "status": {
+          "$ref": "#/definitions/HealthCheckResponseServingStatus"
+        }
+      }
+    }
+  }
+}