Merge branch 'master' into codecov

cmd/healthcheck/main.go

@@ -0,0 +1,44 @@
+// Copyright 2020 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// healthcheck makes a HTTP(S) request to the supplied url and exits with 0
+// (success) if the HTTP response code was in the 2xx range or 1 (unhealthy).
+//
+// TLS certificate errors are ignored in order to support self-signed certs.
+package main
+
+import (
+	"crypto/tls"
+	"log"
+	"net/http"
+	"os"
+)
+
+func main() {
+	if len(os.Args) < 2 {
+		log.Fatal("Expected URL as command-line argument")
+	}
+	url := os.Args[1]
+	//nolint:gas
+	client := &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}}
+
+	//nolint:gas
+	resp, err := client.Get(url)
+	if err != nil {
+		log.Fatalf("http.Get(%v): %v", url, err)
+	}
+	if resp.StatusCode < 200 || resp.StatusCode > 299 {
+		log.Fatalf("HTTP status %v was not in the 2xx range", resp.StatusCode)
+	}
+}

cmd/keytransparency-monitor/Dockerfile

@@ -7,11 +7,14 @@ RUN go mod download
 COPY . .
 
 RUN go get ./cmd/keytransparency-monitor
+RUN go get ./cmd/healthcheck
 
 FROM gcr.io/distroless/base
 
 COPY --from=build /go/bin/keytransparency-monitor /
+COPY --from=build /go/bin/healthcheck /
 
 ENTRYPOINT ["/keytransparency-monitor"]
+HEALTHCHECK CMD ["/healthcheck","https://localhost:8099/healthz"]
 
 EXPOSE 8099

cmd/keytransparency-monitor/main.go

@@ -21,19 +21,22 @@ import (
 	"flag"
 	"net"
 	"net/http"
+	"time"
 
 	"github.com/golang/glog"
 	tcrypto "github.com/google/trillian/crypto"
 	"github.com/google/trillian/crypto/keys/pem"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/credentials"
 	"google.golang.org/grpc/reflection"
 
 	"github.com/google/keytransparency/cmd/serverutil"
 	"github.com/google/keytransparency/core/fake"
 	"github.com/google/keytransparency/core/monitor"
 	"github.com/google/keytransparency/core/monitorserver"
+	"github.com/google/keytransparency/internal/backoff"
 
 	mopb "github.com/google/keytransparency/core/api/monitor/v1/monitor_go_proto"
 	pb "github.com/google/keytransparency/core/api/v1/keytransparency_go_proto"
@@ -47,7 +50,7 @@ var (
 
 	signingKey         = flag.String("sign-key", "genfiles/monitor_sign-key.pem", "Path to private key PEM for SMH signing")
 	signingKeyPassword = flag.String("password", "towel", "Password of the private key PEM file for SMH signing")
-	ktURL              = flag.String("kt-url", "localhost:8080", "URL of key-server.")
+	ktURL              = flag.String("kt-url", "localhost:443", "URL of key-server.")
 	insecure           = flag.Bool("insecure", false, "Skip TLS checks")
 	directoryID        = flag.String("directoryid", "", "KT Directory identifier to monitor")
 
@@ -66,8 +69,22 @@ func main() {
 	}
 	ktClient := pb.NewKeyTransparencyClient(cc)
 
-	config, err := ktClient.GetDirectory(ctx, &pb.GetDirectoryRequest{DirectoryId: *directoryID})
-	if err != nil {
+	// The first gRPC command might fail while the keyserver is starting up. Retry for up to 1 minute.
+	cctx, cancel := context.WithTimeout(ctx, time.Minute)
+	defer cancel()
+	b := backoff.Backoff{
+		Min:    time.Millisecond,
+		Max:    time.Second,
+		Factor: 1.5,
+	}
+	var config *pb.Directory
+	if err := b.Retry(cctx, func() (err error) {
+		config, err = ktClient.GetDirectory(ctx, &pb.GetDirectoryRequest{DirectoryId: *directoryID})
+		if err != nil {
+			glog.Errorf("GetDirectory(%v/%v): %v", *ktURL, *directoryID, err)
+		}
+		return
+	}, codes.Unavailable); err != nil {
 		glog.Exitf("Could not read directory info %v:", err)
 	}
 
@@ -124,7 +141,9 @@ func main() {
 	// Insert handlers for other http paths here.
 	mux := http.NewServeMux()
 	mux.Handle("/metrics", promhttp.Handler())
-	mux.Handle("/", gwmux)
+	mux.Handle("/healthz", serverutil.Healthz())
+	mux.Handle("/readyz", serverutil.Healthz())
+	mux.Handle("/", serverutil.RootHealthHandler(gwmux))
 
 	// Serve HTTP2 server over TLS.
 	glog.Infof("Listening on %v", *addr)

cmd/keytransparency-sequencer/Dockerfile

@@ -7,9 +7,15 @@ RUN go mod download
 COPY . .
 
 RUN go get ./cmd/keytransparency-sequencer
+RUN go get ./cmd/healthcheck
 
 FROM gcr.io/distroless/base
 
 COPY --from=build /go/bin/keytransparency-sequencer /
+COPY --from=build /go/bin/healthcheck /
 
 ENTRYPOINT ["/keytransparency-sequencer"]
+HEALTHCHECK CMD ["/healthcheck","http://localhost:8081/healthz"]
+
+EXPOSE 8080
+EXPOSE 8081

cmd/keytransparency-sequencer/main.go

@@ -185,7 +185,7 @@ func main() {
 	glog.Infof("Signer starting")
 
 	// Run servers
-	go serveHTTPMetric(*metricsAddr)
+	go serveHTTPMetrics(*metricsAddr, sqldb)
 	go serveHTTPGateway(ctx, lis, dopts, grpcServer,
 		pb.RegisterKeyTransparencyAdminHandlerFromEndpoint,
 	)

cmd/keytransparency-sequencer/server.go

@@ -16,6 +16,7 @@ package main
 
 import (
 	"context"
+	"database/sql"
 	"net"
 	"net/http"
 
@@ -26,12 +27,15 @@ import (
 	"google.golang.org/grpc"
 )
 
-func serveHTTPMetric(addr string) {
-	metricMux := http.NewServeMux()
-	metricMux.Handle("/metrics", promhttp.Handler())
+func serveHTTPMetrics(addr string, sqldb *sql.DB) {
+	mux := http.NewServeMux()
+	mux.Handle("/metrics", promhttp.Handler())
+	mux.Handle("/healthz", serverutil.Healthz())
+	mux.Handle("/readyz", serverutil.Readyz(sqldb))
+	mux.Handle("/", serverutil.Healthz())
 
-	glog.Infof("Hosting metrics on %v", addr)
-	if err := http.ListenAndServe(addr, metricMux); err != nil {
+	glog.Infof("Hosting server status and metrics on %v", addr)
+	if err := http.ListenAndServe(addr, mux); err != nil {
 		glog.Fatalf("ListenAndServeTLS(%v): %v", addr, err)
 	}
 }
@@ -45,7 +49,7 @@ func serveHTTPGateway(ctx context.Context, lis net.Listener, dopts []grpc.DialOp
 	}
 
 	mux := http.NewServeMux()
-	mux.Handle("/", gwmux)
+	mux.Handle("/", serverutil.RootHealthHandler(gwmux))
 
 	server := &http.Server{Handler: serverutil.GrpcHandlerFunc(grpcServer, mux)}
 	if err := server.ServeTLS(lis, *certFile, *keyFile); err != nil {

cmd/keytransparency-server/Dockerfile

@@ -7,11 +7,15 @@ RUN go mod download
 COPY . .
 
 RUN go get ./cmd/keytransparency-server
+RUN go get ./cmd/healthcheck
 
 FROM gcr.io/distroless/base
 
 COPY --from=build /go/bin/keytransparency-server /
+COPY --from=build /go/bin/healthcheck /
 
 ENTRYPOINT ["/keytransparency-server"]
+HEALTHCHECK CMD ["/healthcheck","http://localhost:8081/healthz"]
 
 EXPOSE 8080
+EXPOSE 8081

cmd/keytransparency-server/main.go

@@ -152,10 +152,13 @@ func main() {
 
 	// Insert handlers for other http paths here.
 	mux := http.NewServeMux()
-	mux.Handle("/", gwmux)
+	mux.Handle("/", serverutil.RootHealthHandler(gwmux))
 
 	metricMux := http.NewServeMux()
+	metricMux.Handle("/healthz", serverutil.Healthz())
+	metricMux.Handle("/readyz", serverutil.Readyz(sqldb))
 	metricMux.Handle("/metrics", promhttp.Handler())
+	metricMux.Handle("/", serverutil.Healthz())
 	go func() {
 		glog.Infof("Hosting metrics on %v", *metricsAddr)
 		if err := http.ListenAndServe(*metricsAddr, metricMux); err != nil {

cmd/serverutil/healthz.go

@@ -0,0 +1,40 @@
+// Copyright 2020 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package serverutil
+
+import "net/http"
+
+// Healthz is a liveness handler that always responds with HTTP 200.
+func Healthz() http.HandlerFunc { return healthz }
+
+func healthz(w http.ResponseWriter, _ *http.Request) {
+	w.WriteHeader(http.StatusOK)
+	w.Write([]byte("ok"))
+}
+
+// RootHeaalthHandler handles liveness checks at "/".
+// All other requests are passed through to `otherHandler`.
+func RootHealthHandler(otherHandler http.Handler) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		// Capture requests for the root "/" page first.
+		// This is the default load balancer health check.
+		// https://cloud.google.com/kubernetes-engine/docs/concepts/ingress#health_checks
+		if r.URL.Path == "/" {
+			healthz(w, r)
+			return
+		}
+		otherHandler.ServeHTTP(w, r)
+	}
+}

cmd/serverutil/readyz.go

@@ -0,0 +1,31 @@
+// Copyright 2020 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package serverutil
+
+import (
+	"database/sql"
+	"net/http"
+)
+
+// Readyz is a readiness probe.
+func Readyz(db *sql.DB) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		if db == nil || db.PingContext(r.Context()) != nil {
+			http.Error(w, http.StatusText(http.StatusServiceUnavailable), http.StatusServiceUnavailable)
+			return
+		}
+		w.WriteHeader(http.StatusOK)
+	}
+}

cmd/serverutil/serverutil.go

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+// Package serverutil provides helper functions to main.go files.
 package serverutil
 
 import (

deploy/kubernetes/base/db-deployment.yaml

@@ -1,10 +1,6 @@
 apiVersion: apps/v1
 kind: Deployment
 metadata:
-  annotations:
-    kompose.cmd: kompose convert --file ../../docker-compose.yml
-    kompose.version: 1.14.0 ()
-  creationTimestamp: null
   labels:
     io.kompose.service: db
   name: db

deploy/kubernetes/base/db-service.yaml

@@ -1,10 +1,6 @@
 apiVersion: v1
 kind: Service
 metadata:
-  annotations:
-    kompose.cmd: kompose convert --file ../../docker-compose.yml
-    kompose.version: 1.14.0 ()
-  creationTimestamp: null
   labels:
     io.kompose.service: db
   name: db

deploy/kubernetes/base/init-pod.yaml

@@ -1,7 +1,6 @@
 apiVersion: v1
 kind: Pod
 metadata:
-  creationTimestamp: null
   labels:
     io.kompose.service: init
   name: init

deploy/kubernetes/base/log-server-deployment.yaml

@@ -1,10 +1,6 @@
 apiVersion: apps/v1
 kind: Deployment
 metadata:
-  annotations:
-    kompose.cmd: kompose convert --file ../../docker-compose.yml
-    kompose.version: 1.14.0 ()
-  creationTimestamp: null
   labels:
     io.kompose.service: log-server
   name: log-server

deploy/kubernetes/base/log-server-service.yaml

@@ -1,10 +1,6 @@
 apiVersion: v1
 kind: Service
 metadata:
-  annotations:
-    kompose.cmd: kompose convert --file ../../docker-compose.yml
-    kompose.version: 1.14.0 ()
-  creationTimestamp: null
   labels:
     io.kompose.service: log-server
   name: log-server

deploy/kubernetes/base/log-signer-deployment.yaml

@@ -1,10 +1,6 @@
 apiVersion: apps/v1
 kind: Deployment
 metadata:
-  annotations:
-    kompose.cmd: kompose convert --file ../../docker-compose.yml
-    kompose.version: 1.14.0 ()
-  creationTimestamp: null
   labels:
     io.kompose.service: log-signer
   name: log-signer

deploy/kubernetes/base/log-signer-service.yaml

@@ -1,10 +1,6 @@
 apiVersion: v1
 kind: Service
 metadata:
-  annotations:
-    kompose.cmd: kompose convert --file ../../docker-compose.yml
-    kompose.version: 1.14.0 ()
-  creationTimestamp: null
   labels:
     io.kompose.service: log-signer
   name: log-signer

deploy/kubernetes/base/map-server-deployment.yaml

@@ -1,10 +1,6 @@
 apiVersion: apps/v1
 kind: Deployment
 metadata:
-  annotations:
-    kompose.cmd: kompose convert --file ../../docker-compose.yml
-    kompose.version: 1.14.0 ()
-  creationTimestamp: null
   labels:
     io.kompose.service: map-server
   name: map-server

deploy/kubernetes/base/map-server-service.yaml

@@ -1,10 +1,6 @@
 apiVersion: v1
 kind: Service
 metadata:
-  annotations:
-    kompose.cmd: kompose convert --file ../../docker-compose.yml
-    kompose.version: 1.14.0 ()
-  creationTimestamp: null
   labels:
     io.kompose.service: map-server
   name: map-server