From 47d748c5dc989ea46142569bf42636c622fe128a Mon Sep 17 00:00:00 2001 From: bjhaid Date: Sun, 16 Jul 2017 02:00:35 +0000 Subject: [PATCH] This adds an etcd health check endpoint to kube-apiserver addressing https://github.com/kubernetes/kubernetes/issues/48215. --- cmd/kube-apiserver/app/BUILD | 3 +-- cmd/kube-apiserver/app/server.go | 2 +- .../k8s.io/apiserver/pkg/server/options/BUILD | 2 ++ .../apiserver/pkg/server/options/etcd.go | 18 ++++++++++++++++ .../pkg/storage/etcd3}/preflight/BUILD | 13 ------------ .../pkg/storage/etcd3}/preflight/checks.go | 14 ++++++++----- .../storage/etcd3}/preflight/checks_test.go | 21 +++++++++++++++---- 7 files changed, 48 insertions(+), 25 deletions(-) rename {cmd/kube-apiserver/app => staging/src/k8s.io/apiserver/pkg/storage/etcd3}/preflight/BUILD (67%) rename {cmd/kube-apiserver/app => staging/src/k8s.io/apiserver/pkg/storage/etcd3}/preflight/checks.go (78%) rename {cmd/kube-apiserver/app => staging/src/k8s.io/apiserver/pkg/storage/etcd3}/preflight/checks_test.go (80%) diff --git a/cmd/kube-apiserver/app/BUILD b/cmd/kube-apiserver/app/BUILD index 8cebb69e06687..44a91ed06a1f4 100644 --- a/cmd/kube-apiserver/app/BUILD +++ b/cmd/kube-apiserver/app/BUILD @@ -18,7 +18,6 @@ go_library( tags = ["automanaged"], deps = [ "//cmd/kube-apiserver/app/options:go_default_library", - "//cmd/kube-apiserver/app/preflight:go_default_library", "//pkg/api:go_default_library", "//pkg/apis/apps:go_default_library", "//pkg/apis/batch:go_default_library", @@ -94,6 +93,7 @@ go_library( "//vendor/k8s.io/apiserver/pkg/server/options:go_default_library", "//vendor/k8s.io/apiserver/pkg/server/options/encryptionconfig:go_default_library", "//vendor/k8s.io/apiserver/pkg/server/storage:go_default_library", + "//vendor/k8s.io/apiserver/pkg/storage/etcd3/preflight:go_default_library", "//vendor/k8s.io/client-go/informers:go_default_library", "//vendor/k8s.io/client-go/kubernetes:go_default_library", "//vendor/k8s.io/kube-aggregator/pkg/apis/apiregistration:go_default_library", @@ -116,7 +116,6 @@ filegroup( srcs = [ ":package-srcs", "//cmd/kube-apiserver/app/options:all-srcs", - "//cmd/kube-apiserver/app/preflight:all-srcs", "//cmd/kube-apiserver/app/testing:all-srcs", ], tags = ["automanaged"], diff --git a/cmd/kube-apiserver/app/server.go b/cmd/kube-apiserver/app/server.go index a8e8356b4c9e2..500dcba37e35e 100644 --- a/cmd/kube-apiserver/app/server.go +++ b/cmd/kube-apiserver/app/server.go @@ -54,11 +54,11 @@ import ( aggregatorapiserver "k8s.io/kube-aggregator/pkg/apiserver" //aggregatorinformers "k8s.io/kube-aggregator/pkg/client/informers/internalversion" + "k8s.io/apiserver/pkg/storage/etcd3/preflight" clientgoinformers "k8s.io/client-go/informers" clientgoclientset "k8s.io/client-go/kubernetes" clientset "k8s.io/client-go/kubernetes" "k8s.io/kubernetes/cmd/kube-apiserver/app/options" - "k8s.io/kubernetes/cmd/kube-apiserver/app/preflight" "k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/apis/apps" "k8s.io/kubernetes/pkg/apis/batch" diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/BUILD b/staging/src/k8s.io/apiserver/pkg/server/options/BUILD index 999afcd2e8a60..0bbcc6d74a073 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/BUILD +++ b/staging/src/k8s.io/apiserver/pkg/server/options/BUILD @@ -62,7 +62,9 @@ go_library( "//vendor/k8s.io/apiserver/pkg/registry/generic:go_default_library", "//vendor/k8s.io/apiserver/pkg/registry/generic/registry:go_default_library", "//vendor/k8s.io/apiserver/pkg/server:go_default_library", + "//vendor/k8s.io/apiserver/pkg/server/healthz:go_default_library", "//vendor/k8s.io/apiserver/pkg/server/storage:go_default_library", + "//vendor/k8s.io/apiserver/pkg/storage/etcd3/preflight:go_default_library", "//vendor/k8s.io/apiserver/pkg/storage/storagebackend:go_default_library", "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library", "//vendor/k8s.io/apiserver/pkg/util/flag:go_default_library", diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/etcd.go b/staging/src/k8s.io/apiserver/pkg/server/options/etcd.go index 16a111397ac0f..16e2626de4800 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/etcd.go +++ b/staging/src/k8s.io/apiserver/pkg/server/options/etcd.go @@ -18,6 +18,7 @@ package options import ( "fmt" + "net/http" "github.com/spf13/pflag" @@ -26,7 +27,9 @@ import ( "k8s.io/apiserver/pkg/registry/generic" genericregistry "k8s.io/apiserver/pkg/registry/generic/registry" "k8s.io/apiserver/pkg/server" + "k8s.io/apiserver/pkg/server/healthz" serverstorage "k8s.io/apiserver/pkg/server/storage" + "k8s.io/apiserver/pkg/storage/etcd3/preflight" "k8s.io/apiserver/pkg/storage/storagebackend" ) @@ -127,15 +130,30 @@ func (s *EtcdOptions) AddFlags(fs *pflag.FlagSet) { } func (s *EtcdOptions) ApplyTo(c *server.Config) error { + s.addEtcdHealthEndpoint(c) c.RESTOptionsGetter = &SimpleRestOptionsFactory{Options: *s} return nil } func (s *EtcdOptions) ApplyWithStorageFactoryTo(factory serverstorage.StorageFactory, c *server.Config) error { + s.addEtcdHealthEndpoint(c) c.RESTOptionsGetter = &storageFactoryRestOptionsFactory{Options: *s, StorageFactory: factory} return nil } +func (s *EtcdOptions) addEtcdHealthEndpoint(c *server.Config) { + c.HealthzChecks = append(c.HealthzChecks, healthz.NamedCheck("etcd", func(r *http.Request) error { + done, err := preflight.EtcdConnection{ServerList: s.StorageConfig.ServerList}.CheckEtcdServers() + if !done { + return fmt.Errorf("etcd failed") + } + if err != nil { + return err + } + return nil + })) +} + type SimpleRestOptionsFactory struct { Options EtcdOptions } diff --git a/cmd/kube-apiserver/app/preflight/BUILD b/staging/src/k8s.io/apiserver/pkg/storage/etcd3/preflight/BUILD similarity index 67% rename from cmd/kube-apiserver/app/preflight/BUILD rename to staging/src/k8s.io/apiserver/pkg/storage/etcd3/preflight/BUILD index 77c42c7d053fb..ff93f24f07bb3 100644 --- a/cmd/kube-apiserver/app/preflight/BUILD +++ b/staging/src/k8s.io/apiserver/pkg/storage/etcd3/preflight/BUILD @@ -21,16 +21,3 @@ go_test( tags = ["automanaged"], deps = ["//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library"], ) - -filegroup( - name = "package-srcs", - srcs = glob(["**"]), - tags = ["automanaged"], - visibility = ["//visibility:private"], -) - -filegroup( - name = "all-srcs", - srcs = [":package-srcs"], - tags = ["automanaged"], -) diff --git a/cmd/kube-apiserver/app/preflight/checks.go b/staging/src/k8s.io/apiserver/pkg/storage/etcd3/preflight/checks.go similarity index 78% rename from cmd/kube-apiserver/app/preflight/checks.go rename to staging/src/k8s.io/apiserver/pkg/storage/etcd3/preflight/checks.go index c3bbd0577909f..899a6fa89c4e7 100644 --- a/cmd/kube-apiserver/app/preflight/checks.go +++ b/staging/src/k8s.io/apiserver/pkg/storage/etcd3/preflight/checks.go @@ -36,20 +36,24 @@ type EtcdConnection struct { ServerList []string } -func (EtcdConnection) serverReachable(address string) bool { - if conn, err := net.DialTimeout("tcp", address, connectionTimeout); err == nil { +func (EtcdConnection) serverReachable(connURL *url.URL) bool { + scheme := connURL.Scheme + if scheme == "http" || scheme == "https" || scheme == "tcp" { + scheme = "tcp" + } + if conn, err := net.DialTimeout(scheme, connURL.Host, connectionTimeout); err == nil { defer conn.Close() return true } return false } -func parseServerURI(serverURI string) (string, error) { +func parseServerURI(serverURI string) (*url.URL, error) { connURL, err := url.Parse(serverURI) if err != nil { - return "", fmt.Errorf("unable to parse etcd url: %v", err) + return &url.URL{}, fmt.Errorf("unable to parse etcd url: %v", err) } - return connURL.Host, nil + return connURL, nil } // CheckEtcdServers will attempt to reach all etcd servers once. If any diff --git a/cmd/kube-apiserver/app/preflight/checks_test.go b/staging/src/k8s.io/apiserver/pkg/storage/etcd3/preflight/checks_test.go similarity index 80% rename from cmd/kube-apiserver/app/preflight/checks_test.go rename to staging/src/k8s.io/apiserver/pkg/storage/etcd3/preflight/checks_test.go index da6ad0f3aca3e..e3295725885a0 100644 --- a/cmd/kube-apiserver/app/preflight/checks_test.go +++ b/staging/src/k8s.io/apiserver/pkg/storage/etcd3/preflight/checks_test.go @@ -17,6 +17,7 @@ limitations under the License. package preflight import ( + "net/url" "testing" "time" @@ -24,14 +25,26 @@ import ( ) func TestParseServerURIGood(t *testing.T) { - host, err := parseServerURI("https://127.0.0.1:2379") + connURL, err := parseServerURI("https://127.0.0.1:2379") if err != nil { t.Fatalf("unexpected error: %v", err) } reference := "127.0.0.1:2379" - if host != reference { - t.Fatalf("server uri was not parsed correctly, host %s was invalid", host) + if connURL.Host != reference { + t.Fatalf("server uri was not parsed correctly, host %s was invalid", connURL.Host) + } +} + +func TestParseServerURIGoodUnix(t *testing.T) { + connURL, err := parseServerURI("unix://127.0.0.1:21002112605") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + reference := "127.0.0.1:21002112605" + if connURL.Host != reference { + t.Fatalf("server uri was not parsed correctly, host %s was invalid", connURL.Host) } } @@ -45,7 +58,7 @@ func TestParseServerURIBad(t *testing.T) { func TestEtcdConnection(t *testing.T) { etcd := new(EtcdConnection) - result := etcd.serverReachable("-not a real network address-") + result := etcd.serverReachable(&url.URL{Host: "-not a real network address-", Scheme: "tcp"}) if result { t.Fatal("checkConnection should not have succeeded") }