From 00c60ddff60abb33350be63569da1a1d087ea38f Mon Sep 17 00:00:00 2001 From: John Gardiner Myers Date: Wed, 15 Jul 2020 21:36:43 -0700 Subject: [PATCH 01/13] Add server code to kops-controller --- cmd/kops-controller/BUILD.bazel | 3 +- cmd/kops-controller/main.go | 13 +++ cmd/kops-controller/pkg/config/options.go | 15 +++- cmd/kops-controller/pkg/server/BUILD.bazel | 9 ++ cmd/kops-controller/pkg/server/server.go | 47 ++++++++++ hack/.packages | 1 + nodeup/pkg/model/BUILD.bazel | 3 + nodeup/pkg/model/context.go | 6 ++ nodeup/pkg/model/kops_controller.go | 85 +++++++++++++++++++ nodeup/pkg/model/kops_controller_test.go | 30 +++++++ pkg/apis/kops/model/BUILD.bazel | 5 +- pkg/apis/kops/model/features.go | 26 ++++++ pkg/model/context.go | 5 ++ pkg/wellknownports/wellknownports.go | 3 + pkg/wellknownusers/wellknownusers.go | 9 +- upup/models/bindata.go | 9 +- .../k8s-1.16.yaml.template | 9 +- upup/pkg/fi/cloudup/template_functions.go | 12 ++- upup/pkg/fi/nodeup/command.go | 1 + 19 files changed, 281 insertions(+), 10 deletions(-) create mode 100644 cmd/kops-controller/pkg/server/BUILD.bazel create mode 100644 cmd/kops-controller/pkg/server/server.go create mode 100644 nodeup/pkg/model/kops_controller.go create mode 100644 nodeup/pkg/model/kops_controller_test.go create mode 100644 pkg/apis/kops/model/features.go diff --git a/cmd/kops-controller/BUILD.bazel b/cmd/kops-controller/BUILD.bazel index 756dd7b64c6f6..a5172a9fd612e 100644 --- a/cmd/kops-controller/BUILD.bazel +++ b/cmd/kops-controller/BUILD.bazel @@ -8,6 +8,7 @@ go_library( deps = [ "//cmd/kops-controller/controllers:go_default_library", "//cmd/kops-controller/pkg/config:go_default_library", + "//cmd/kops-controller/pkg/server:go_default_library", "//pkg/nodeidentity:go_default_library", "//pkg/nodeidentity/aws:go_default_library", "//pkg/nodeidentity/do:go_default_library", @@ -47,7 +48,7 @@ ARCH = [ architecture = arch, base = "@distroless_base//image", cmd = ["/kops-controller"], - user = "10001", + user = "10011", files = [ "//cmd/kops-controller", ], diff --git a/cmd/kops-controller/main.go b/cmd/kops-controller/main.go index c1a4a7fd2c3f1..d2e8f55760077 100644 --- a/cmd/kops-controller/main.go +++ b/cmd/kops-controller/main.go @@ -29,6 +29,7 @@ import ( "k8s.io/klog/klogr" "k8s.io/kops/cmd/kops-controller/controllers" "k8s.io/kops/cmd/kops-controller/pkg/config" + "k8s.io/kops/cmd/kops-controller/pkg/server" "k8s.io/kops/pkg/nodeidentity" nodeidentityaws "k8s.io/kops/pkg/nodeidentity/aws" nodeidentitydo "k8s.io/kops/pkg/nodeidentity/do" @@ -81,6 +82,18 @@ func main() { } ctrl.SetLogger(klogr.New()) + if opt.Server != nil { + srv, err := server.NewServer(&opt) + if err != nil { + setupLog.Error(err, "unable to create server") + os.Exit(1) + } + go func() { + err := srv.Start() + setupLog.Error(err, "unable to start server") + os.Exit(1) + }() + } if err := buildScheme(); err != nil { setupLog.Error(err, "error building scheme") diff --git a/cmd/kops-controller/pkg/config/options.go b/cmd/kops-controller/pkg/config/options.go index 22a15914b17ca..fa98e935f3175 100644 --- a/cmd/kops-controller/pkg/config/options.go +++ b/cmd/kops-controller/pkg/config/options.go @@ -17,9 +17,20 @@ limitations under the License. package config type Options struct { - Cloud string `json:"cloud,omitempty"` - ConfigBase string `json:"configBase,omitempty"` + Cloud string `json:"cloud,omitempty"` + ConfigBase string `json:"configBase,omitempty"` + Server *ServerOptions `json:"server,omitempty"` } func (o *Options) PopulateDefaults() { } + +type ServerOptions struct { + // Listen is the network endpoint (ip and port) we should listen on. + Listen string + + // ServerKeyPath is the path to our TLS serving private key. + ServerKeyPath string `json:"serverKeyPath,omitempty"` + // ServerCertificatePath is the path to our TLS serving certificate. + ServerCertificatePath string `json:"serverCertificatePath,omitempty"` +} diff --git a/cmd/kops-controller/pkg/server/BUILD.bazel b/cmd/kops-controller/pkg/server/BUILD.bazel new file mode 100644 index 0000000000000..831fee41cdb05 --- /dev/null +++ b/cmd/kops-controller/pkg/server/BUILD.bazel @@ -0,0 +1,9 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "go_default_library", + srcs = ["server.go"], + importpath = "k8s.io/kops/cmd/kops-controller/pkg/server", + visibility = ["//visibility:public"], + deps = ["//cmd/kops-controller/pkg/config:go_default_library"], +) diff --git a/cmd/kops-controller/pkg/server/server.go b/cmd/kops-controller/pkg/server/server.go new file mode 100644 index 0000000000000..50cc46b617c8a --- /dev/null +++ b/cmd/kops-controller/pkg/server/server.go @@ -0,0 +1,47 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package server + +import ( + "crypto/tls" + "net/http" + + "k8s.io/kops/cmd/kops-controller/pkg/config" +) + +type Server struct { + opt *config.Options + server *http.Server +} + +func NewServer(opt *config.Options) (*Server, error) { + server := &http.Server{ + Addr: opt.Server.Listen, + TLSConfig: &tls.Config{ + MinVersion: tls.VersionTLS12, + PreferServerCipherSuites: true, + }, + } + return &Server{ + opt: opt, + server: server, + }, nil +} + +func (s *Server) Start() error { + return s.server.ListenAndServeTLS(s.opt.Server.ServerCertificatePath, s.opt.Server.ServerKeyPath) +} diff --git a/hack/.packages b/hack/.packages index 0c713ab385b96..baaa194f367a7 100644 --- a/hack/.packages +++ b/hack/.packages @@ -20,6 +20,7 @@ k8s.io/kops/cmd/kops/util k8s.io/kops/cmd/kops-controller k8s.io/kops/cmd/kops-controller/controllers k8s.io/kops/cmd/kops-controller/pkg/config +k8s.io/kops/cmd/kops-controller/pkg/server k8s.io/kops/cmd/kube-apiserver-healthcheck k8s.io/kops/cmd/nodeup k8s.io/kops/dns-controller/cmd/dns-controller diff --git a/nodeup/pkg/model/BUILD.bazel b/nodeup/pkg/model/BUILD.bazel index dbd483a3fa5ac..ac26b90d5047d 100644 --- a/nodeup/pkg/model/BUILD.bazel +++ b/nodeup/pkg/model/BUILD.bazel @@ -15,6 +15,7 @@ go_library( "file_assets.go", "firewall.go", "hooks.go", + "kops_controller.go", "kube_apiserver.go", "kube_apiserver_healthcheck.go", "kube_controller_manager.go", @@ -41,6 +42,7 @@ go_library( "//nodeup/pkg/distros:go_default_library", "//nodeup/pkg/model/resources:go_default_library", "//pkg/apis/kops:go_default_library", + "//pkg/apis/kops/model:go_default_library", "//pkg/apis/kops/util:go_default_library", "//pkg/apis/nodeup:go_default_library", "//pkg/assets:go_default_library", @@ -86,6 +88,7 @@ go_test( "containerd_test.go", "docker_test.go", "fakes_test.go", + "kops_controller_test.go", "kube_apiserver_test.go", "kube_controller_manager_test.go", "kube_proxy_test.go", diff --git a/nodeup/pkg/model/context.go b/nodeup/pkg/model/context.go index ecead295987a3..5cd801792b928 100644 --- a/nodeup/pkg/model/context.go +++ b/nodeup/pkg/model/context.go @@ -25,6 +25,7 @@ import ( "k8s.io/klog" "k8s.io/kops/nodeup/pkg/distros" "k8s.io/kops/pkg/apis/kops" + "k8s.io/kops/pkg/apis/kops/model" "k8s.io/kops/pkg/apis/kops/util" "k8s.io/kops/pkg/apis/nodeup" "k8s.io/kops/pkg/systemd" @@ -326,6 +327,11 @@ func (c *NodeupModelContext) UseEtcdTLSAuth() bool { return false } +// UseKopsControllerForNodeBootstrap checks if nodeup should use kops-controller to bootstrap. +func (c *NodeupModelContext) UseKopsControllerForNodeBootstrap() bool { + return model.UseKopsControllerForNodeBootstrap(c.Cluster) +} + // UseNodeAuthorization checks if have a node authorization policy func (c *NodeupModelContext) UseNodeAuthorization() bool { return c.Cluster.Spec.NodeAuthorization != nil diff --git a/nodeup/pkg/model/kops_controller.go b/nodeup/pkg/model/kops_controller.go new file mode 100644 index 0000000000000..1a27af5e85b5e --- /dev/null +++ b/nodeup/pkg/model/kops_controller.go @@ -0,0 +1,85 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package model + +import ( + "path/filepath" + + "k8s.io/kops/pkg/wellknownusers" + "k8s.io/kops/upup/pkg/fi" + "k8s.io/kops/upup/pkg/fi/nodeup/nodetasks" +) + +// KopsControllerBuilder installs the keys for a kops-controller. +type KopsControllerBuilder struct { + *NodeupModelContext +} + +var _ fi.ModelBuilder = &KopsControllerBuilder{} + +// Build is responsible for configuring keys that will be used by kops-controller (via hostPath) +func (b *KopsControllerBuilder) Build(c *fi.ModelBuilderContext) error { + if !b.IsMaster { + return nil + } + + // Create the directory, even if we aren't going to populate it + pkiDir := "/etc/kubernetes/kops-controller" + c.AddTask(&nodetasks.File{ + Path: pkiDir, + Type: nodetasks.FileType_Directory, + Mode: s("0755"), + }) + + if !b.UseKopsControllerForNodeBootstrap() { + return nil + } + + // We run kops-controller under an unprivileged user (wellknownusers.KopsControllerID), and then grant specific permissions + c.AddTask(&nodetasks.UserTask{ + Name: wellknownusers.KopsControllerName, + UID: wellknownusers.KopsControllerID, + Shell: "/sbin/nologin", + }) + + issueCert := &nodetasks.IssueCert{ + Name: "kops-controller", + Signer: fi.CertificateIDCA, + Type: "server", + Subject: nodetasks.PKIXName{CommonName: "kops-controller"}, + AlternateNames: []string{b.Cluster.Spec.MasterInternalName}, + } + c.AddTask(issueCert) + + certResource, keyResource, _ := issueCert.GetResources() + c.AddTask(&nodetasks.File{ + Path: filepath.Join(pkiDir, "kops-controller.crt"), + Contents: certResource, + Type: nodetasks.FileType_File, + Mode: s("0644"), + Owner: s(wellknownusers.KopsControllerName), + }) + c.AddTask(&nodetasks.File{ + Path: filepath.Join(pkiDir, "kops-controller.key"), + Contents: keyResource, + Type: nodetasks.FileType_File, + Mode: s("0600"), + Owner: s(wellknownusers.KopsControllerName), + }) + + return nil +} diff --git a/nodeup/pkg/model/kops_controller_test.go b/nodeup/pkg/model/kops_controller_test.go new file mode 100644 index 0000000000000..46c64e96428a3 --- /dev/null +++ b/nodeup/pkg/model/kops_controller_test.go @@ -0,0 +1,30 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package model + +import ( + "testing" + + "k8s.io/kops/upup/pkg/fi" +) + +func TestKopsControllerBuilder(t *testing.T) { + RunGoldenTest(t, "tests/golden/minimal", "kops-controller", func(nodeupModelContext *NodeupModelContext, target *fi.ModelBuilderContext) error { + builder := KopsControllerBuilder{NodeupModelContext: nodeupModelContext} + return builder.Build(target) + }) +} diff --git a/pkg/apis/kops/model/BUILD.bazel b/pkg/apis/kops/model/BUILD.bazel index 395598832bfb3..dd9a0e9503d7c 100644 --- a/pkg/apis/kops/model/BUILD.bazel +++ b/pkg/apis/kops/model/BUILD.bazel @@ -2,7 +2,10 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") go_library( name = "go_default_library", - srcs = ["utils.go"], + srcs = [ + "features.go", + "utils.go", + ], importpath = "k8s.io/kops/pkg/apis/kops/model", visibility = ["//visibility:public"], deps = [ diff --git a/pkg/apis/kops/model/features.go b/pkg/apis/kops/model/features.go new file mode 100644 index 0000000000000..fbc0836986d82 --- /dev/null +++ b/pkg/apis/kops/model/features.go @@ -0,0 +1,26 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package model + +import ( + "k8s.io/kops/pkg/apis/kops" +) + +// UseKopsControllerForNodeBootstrap is true if nodeup should use kops-controller for bootstrapping. +func UseKopsControllerForNodeBootstrap(cluster *kops.Cluster) bool { + return kops.CloudProviderID(cluster.Spec.CloudProvider) == kops.CloudProviderAWS && cluster.IsKubernetesGTE("1.19") +} diff --git a/pkg/model/context.go b/pkg/model/context.go index e3d9d26341ca4..25ab1ee78f6cd 100644 --- a/pkg/model/context.go +++ b/pkg/model/context.go @@ -267,6 +267,11 @@ func (m *KopsModelContext) CloudTags(name string, shared bool) map[string]string return tags } +// UseKopsControllerForNodeBootstrap checks if nodeup should use kops-controller to bootstrap. +func (m *KopsModelContext) UseKopsControllerForNodeBootstrap() bool { + return model.UseKopsControllerForNodeBootstrap(m.Cluster) +} + // UseBootstrapTokens checks if bootstrap tokens are enabled func (m *KopsModelContext) UseBootstrapTokens() bool { if m.Cluster.Spec.KubeAPIServer == nil { diff --git a/pkg/wellknownports/wellknownports.go b/pkg/wellknownports/wellknownports.go index 71c31b7c790f5..03b7912aeacca 100644 --- a/pkg/wellknownports/wellknownports.go +++ b/pkg/wellknownports/wellknownports.go @@ -44,6 +44,9 @@ const ( // DNSControllerGossipMemberlist is the port where dns-controller listens for the memberlist-backed gossip DNSControllerGossipMemberlist = 3993 + // KopsControllerPort is the port where kops-controller listens. + KopsControllerPort = 3992 + // 4001 is etcd main, 4002 is etcd events, 4003 is etcd cilium // KubeAPIServerHealthCheck is the port where kube-apiserver-healthcheck listens. diff --git a/pkg/wellknownusers/wellknownusers.go b/pkg/wellknownusers/wellknownusers.go index 4e9580a4b6fd2..cc470873a6cd9 100644 --- a/pkg/wellknownusers/wellknownusers.go +++ b/pkg/wellknownusers/wellknownusers.go @@ -21,7 +21,7 @@ package wellknownusers const ( // Generic is the user id we use for non-privileged containers, where we don't need extra permissions - // Used by e.g. dns-controller, kops-controller + // Used by e.g. dns-controller Generic = 10001 // LegacyEtcd is the user id for the etcd user under the legacy provider @@ -30,6 +30,13 @@ const ( // AWSAuthenticator is the user-id for the aws-iam-authenticator (built externally) AWSAuthenticator = 10000 + // KopsControllerID is the user id for kops-controller, which needs some extra permissions e.g. to write local logs + // This should match the user in cmd/kops-controller/BUILD.bazel + KopsControllerID = 10011 + + // KopsControllerName is the username for the kops-controller user + KopsControllerName = "kops-controller" + // KubeApiserverHealthcheckID is the user id for kube-apiserver-healthcheck sidecar // The user needs some extra permissions e.g. to read local secrets // This should match the user in cmd/kube-apiserver-healthcheck/BUILD.bazel diff --git a/upup/models/bindata.go b/upup/models/bindata.go index ae4bb1e2dcc4f..b66390f731586 100644 --- a/upup/models/bindata.go +++ b/upup/models/bindata.go @@ -2536,8 +2536,10 @@ spec: name: etc-ssl-certs readOnly: true {{ end }} - - mountPath: /etc/kubernetes/kops-controller/ + - mountPath: /etc/kubernetes/kops-controller/config/ name: kops-controller-config + - mountPath: /etc/kubernetes/kops-controller/pki/ + name: kops-controller-pki command: {{ range $arg := KopsControllerArgv }} - "{{ $arg }}" @@ -2565,7 +2567,10 @@ spec: - name: kops-controller-config configMap: name: kops-controller - + - name: kops-controller-pki + hostPath: + path: /etc/kubernetes/kops-controller/ + type: Directory --- apiVersion: v1 diff --git a/upup/models/cloudup/resources/addons/kops-controller.addons.k8s.io/k8s-1.16.yaml.template b/upup/models/cloudup/resources/addons/kops-controller.addons.k8s.io/k8s-1.16.yaml.template index 5d4a91fd1c80a..4866f9302bca1 100644 --- a/upup/models/cloudup/resources/addons/kops-controller.addons.k8s.io/k8s-1.16.yaml.template +++ b/upup/models/cloudup/resources/addons/kops-controller.addons.k8s.io/k8s-1.16.yaml.template @@ -53,8 +53,10 @@ spec: name: etc-ssl-certs readOnly: true {{ end }} - - mountPath: /etc/kubernetes/kops-controller/ + - mountPath: /etc/kubernetes/kops-controller/config/ name: kops-controller-config + - mountPath: /etc/kubernetes/kops-controller/pki/ + name: kops-controller-pki command: {{ range $arg := KopsControllerArgv }} - "{{ $arg }}" @@ -82,7 +84,10 @@ spec: - name: kops-controller-config configMap: name: kops-controller - + - name: kops-controller-pki + hostPath: + path: /etc/kubernetes/kops-controller/ + type: Directory --- apiVersion: v1 diff --git a/upup/pkg/fi/cloudup/template_functions.go b/upup/pkg/fi/cloudup/template_functions.go index 2b9ee91485fc0..1430c8299a88f 100644 --- a/upup/pkg/fi/cloudup/template_functions.go +++ b/upup/pkg/fi/cloudup/template_functions.go @@ -31,6 +31,7 @@ import ( "encoding/json" "fmt" "os" + "path" "strconv" "strings" "text/template" @@ -379,6 +380,15 @@ func (tf *TemplateFunctions) KopsControllerConfig() (string, error) { ConfigBase: cluster.Spec.ConfigBase, } + if tf.UseKopsControllerForNodeBootstrap() { + pkiDir := "/etc/kubernetes/kops-controller/pki" + config.Server = &kopscontrollerconfig.ServerOptions{ + Listen: fmt.Sprintf(":%d", wellknownports.KopsControllerPort), + ServerCertificatePath: path.Join(pkiDir, "kops-controller.crt"), + ServerKeyPath: path.Join(pkiDir, "kops-controller.key"), + } + } + // To avoid indentation problems, we marshal as json. json is a subset of yaml b, err := json.Marshal(config) if err != nil { @@ -397,7 +407,7 @@ func (tf *TemplateFunctions) KopsControllerArgv() ([]string, error) { // Verbose, but not excessive logging argv = append(argv, "--v=2") - argv = append(argv, "--conf=/etc/kubernetes/kops-controller/config.yaml") + argv = append(argv, "--conf=/etc/kubernetes/kops-controller/config/config.yaml") return argv, nil } diff --git a/upup/pkg/fi/nodeup/command.go b/upup/pkg/fi/nodeup/command.go index 43df77a21d2dc..0772345fc6166 100644 --- a/upup/pkg/fi/nodeup/command.go +++ b/upup/pkg/fi/nodeup/command.go @@ -244,6 +244,7 @@ func (c *NodeUpCommand) Run(out io.Writer) error { loader.Builders = append(loader.Builders, &model.KubeSchedulerBuilder{NodeupModelContext: modelContext}) loader.Builders = append(loader.Builders, &model.EtcdManagerTLSBuilder{NodeupModelContext: modelContext}) loader.Builders = append(loader.Builders, &model.KubeProxyBuilder{NodeupModelContext: modelContext}) + loader.Builders = append(loader.Builders, &model.KopsControllerBuilder{NodeupModelContext: modelContext}) loader.Builders = append(loader.Builders, &networking.CommonBuilder{NodeupModelContext: modelContext}) loader.Builders = append(loader.Builders, &networking.CalicoBuilder{NodeupModelContext: modelContext}) From 82c75211cfe2edf707a04a30c99d884c4cce2cd3 Mon Sep 17 00:00:00 2001 From: John Gardiner Myers Date: Wed, 15 Jul 2020 21:42:04 -0700 Subject: [PATCH 02/13] update-expected.sh --- .../tests/golden/minimal/tasks-kops-controller.yaml | 3 +++ .../bootstrapchannelbuilder/amazonvpc/manifest.yaml | 2 +- .../awsiamauthenticator/manifest.yaml | 2 +- .../cilium/kops-controller.addons.k8s.io-k8s-1.16.yaml | 10 ++++++++-- .../tests/bootstrapchannelbuilder/cilium/manifest.yaml | 2 +- .../simple/kops-controller.addons.k8s.io-k8s-1.16.yaml | 10 ++++++++-- .../tests/bootstrapchannelbuilder/simple/manifest.yaml | 2 +- .../tests/bootstrapchannelbuilder/weave/manifest.yaml | 2 +- 8 files changed, 24 insertions(+), 9 deletions(-) create mode 100644 nodeup/pkg/model/tests/golden/minimal/tasks-kops-controller.yaml diff --git a/nodeup/pkg/model/tests/golden/minimal/tasks-kops-controller.yaml b/nodeup/pkg/model/tests/golden/minimal/tasks-kops-controller.yaml new file mode 100644 index 0000000000000..8e5f9eb837d6d --- /dev/null +++ b/nodeup/pkg/model/tests/golden/minimal/tasks-kops-controller.yaml @@ -0,0 +1,3 @@ +mode: "0755" +path: /etc/kubernetes/kops-controller +type: directory diff --git a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/amazonvpc/manifest.yaml b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/amazonvpc/manifest.yaml index aee0db3dccefa..cd68ede5d9cc9 100644 --- a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/amazonvpc/manifest.yaml +++ b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/amazonvpc/manifest.yaml @@ -7,7 +7,7 @@ spec: - id: k8s-1.16 kubernetesVersion: '>=1.16.0-alpha.0' manifest: kops-controller.addons.k8s.io/k8s-1.16.yaml - manifestHash: c0729abd228804c1083ed62a018e80c2de9c796c + manifestHash: 7a7039ba3b0e9c0027e486902fba3c6d266cbb46 name: kops-controller.addons.k8s.io selector: k8s-addon: kops-controller.addons.k8s.io diff --git a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/awsiamauthenticator/manifest.yaml b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/awsiamauthenticator/manifest.yaml index c8a2a192539d8..4e3f8b9c8d57c 100644 --- a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/awsiamauthenticator/manifest.yaml +++ b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/awsiamauthenticator/manifest.yaml @@ -7,7 +7,7 @@ spec: - id: k8s-1.16 kubernetesVersion: '>=1.16.0-alpha.0' manifest: kops-controller.addons.k8s.io/k8s-1.16.yaml - manifestHash: c0729abd228804c1083ed62a018e80c2de9c796c + manifestHash: 7a7039ba3b0e9c0027e486902fba3c6d266cbb46 name: kops-controller.addons.k8s.io selector: k8s-addon: kops-controller.addons.k8s.io diff --git a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/cilium/kops-controller.addons.k8s.io-k8s-1.16.yaml b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/cilium/kops-controller.addons.k8s.io-k8s-1.16.yaml index cacba3e380dd7..cd2920e40e0aa 100644 --- a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/cilium/kops-controller.addons.k8s.io-k8s-1.16.yaml +++ b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/cilium/kops-controller.addons.k8s.io-k8s-1.16.yaml @@ -35,7 +35,7 @@ spec: - command: - /kops-controller - --v=2 - - --conf=/etc/kubernetes/kops-controller/config.yaml + - --conf=/etc/kubernetes/kops-controller/config/config.yaml image: kope/kops-controller:1.19.0-alpha.2 name: kops-controller resources: @@ -45,8 +45,10 @@ spec: securityContext: runAsNonRoot: true volumeMounts: - - mountPath: /etc/kubernetes/kops-controller/ + - mountPath: /etc/kubernetes/kops-controller/config/ name: kops-controller-config + - mountPath: /etc/kubernetes/kops-controller/pki/ + name: kops-controller-pki dnsPolicy: Default hostNetwork: true nodeSelector: @@ -60,6 +62,10 @@ spec: - configMap: name: kops-controller name: kops-controller-config + - hostPath: + path: /etc/kubernetes/kops-controller/ + type: Directory + name: kops-controller-pki updateStrategy: rollingUpdate: maxUnavailable: 1 diff --git a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/cilium/manifest.yaml b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/cilium/manifest.yaml index e2b912ff5b84b..992b2e201c2c4 100644 --- a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/cilium/manifest.yaml +++ b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/cilium/manifest.yaml @@ -7,7 +7,7 @@ spec: - id: k8s-1.16 kubernetesVersion: '>=1.16.0-alpha.0' manifest: kops-controller.addons.k8s.io/k8s-1.16.yaml - manifestHash: c0729abd228804c1083ed62a018e80c2de9c796c + manifestHash: 7a7039ba3b0e9c0027e486902fba3c6d266cbb46 name: kops-controller.addons.k8s.io selector: k8s-addon: kops-controller.addons.k8s.io diff --git a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/simple/kops-controller.addons.k8s.io-k8s-1.16.yaml b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/simple/kops-controller.addons.k8s.io-k8s-1.16.yaml index cacba3e380dd7..cd2920e40e0aa 100644 --- a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/simple/kops-controller.addons.k8s.io-k8s-1.16.yaml +++ b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/simple/kops-controller.addons.k8s.io-k8s-1.16.yaml @@ -35,7 +35,7 @@ spec: - command: - /kops-controller - --v=2 - - --conf=/etc/kubernetes/kops-controller/config.yaml + - --conf=/etc/kubernetes/kops-controller/config/config.yaml image: kope/kops-controller:1.19.0-alpha.2 name: kops-controller resources: @@ -45,8 +45,10 @@ spec: securityContext: runAsNonRoot: true volumeMounts: - - mountPath: /etc/kubernetes/kops-controller/ + - mountPath: /etc/kubernetes/kops-controller/config/ name: kops-controller-config + - mountPath: /etc/kubernetes/kops-controller/pki/ + name: kops-controller-pki dnsPolicy: Default hostNetwork: true nodeSelector: @@ -60,6 +62,10 @@ spec: - configMap: name: kops-controller name: kops-controller-config + - hostPath: + path: /etc/kubernetes/kops-controller/ + type: Directory + name: kops-controller-pki updateStrategy: rollingUpdate: maxUnavailable: 1 diff --git a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/simple/manifest.yaml b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/simple/manifest.yaml index d9bf2dd63e0cd..7449805e34f1e 100644 --- a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/simple/manifest.yaml +++ b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/simple/manifest.yaml @@ -7,7 +7,7 @@ spec: - id: k8s-1.16 kubernetesVersion: '>=1.16.0-alpha.0' manifest: kops-controller.addons.k8s.io/k8s-1.16.yaml - manifestHash: c0729abd228804c1083ed62a018e80c2de9c796c + manifestHash: 7a7039ba3b0e9c0027e486902fba3c6d266cbb46 name: kops-controller.addons.k8s.io selector: k8s-addon: kops-controller.addons.k8s.io diff --git a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/weave/manifest.yaml b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/weave/manifest.yaml index 8d82d23d290e8..e203dffaa5f13 100644 --- a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/weave/manifest.yaml +++ b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/weave/manifest.yaml @@ -7,7 +7,7 @@ spec: - id: k8s-1.16 kubernetesVersion: '>=1.16.0-alpha.0' manifest: kops-controller.addons.k8s.io/k8s-1.16.yaml - manifestHash: c0729abd228804c1083ed62a018e80c2de9c796c + manifestHash: 7a7039ba3b0e9c0027e486902fba3c6d266cbb46 name: kops-controller.addons.k8s.io selector: k8s-addon: kops-controller.addons.k8s.io From 9c01e1f44d6c7f1584be5e051f6640862a1d3a1c Mon Sep 17 00:00:00 2001 From: John Gardiner Myers Date: Thu, 16 Jul 2020 21:27:23 -0700 Subject: [PATCH 03/13] Send bootstrap query from nodeup to kops-controller --- cmd/kops-controller/pkg/server/BUILD.bazel | 7 +- cmd/kops-controller/pkg/server/server.go | 62 ++++++++- nodeup/pkg/model/BUILD.bazel | 1 + nodeup/pkg/model/bootstrap_client.go | 46 +++++++ pkg/apis/nodeup/BUILD.bazel | 5 +- pkg/apis/nodeup/bootstrap.go | 29 ++++ upup/pkg/fi/nodeup/command.go | 1 + upup/pkg/fi/nodeup/nodetasks/BUILD.bazel | 3 + .../fi/nodeup/nodetasks/bootstrap_client.go | 125 ++++++++++++++++++ 9 files changed, 275 insertions(+), 4 deletions(-) create mode 100644 nodeup/pkg/model/bootstrap_client.go create mode 100644 pkg/apis/nodeup/bootstrap.go create mode 100644 upup/pkg/fi/nodeup/nodetasks/bootstrap_client.go diff --git a/cmd/kops-controller/pkg/server/BUILD.bazel b/cmd/kops-controller/pkg/server/BUILD.bazel index 831fee41cdb05..9c65dc3f40cc0 100644 --- a/cmd/kops-controller/pkg/server/BUILD.bazel +++ b/cmd/kops-controller/pkg/server/BUILD.bazel @@ -5,5 +5,10 @@ go_library( srcs = ["server.go"], importpath = "k8s.io/kops/cmd/kops-controller/pkg/server", visibility = ["//visibility:public"], - deps = ["//cmd/kops-controller/pkg/config:go_default_library"], + deps = [ + "//cmd/kops-controller/pkg/config:go_default_library", + "//pkg/apis/nodeup:go_default_library", + "//vendor/github.com/gorilla/mux:go_default_library", + "//vendor/k8s.io/klog:go_default_library", + ], ) diff --git a/cmd/kops-controller/pkg/server/server.go b/cmd/kops-controller/pkg/server/server.go index 50cc46b617c8a..ad24a90f9a38c 100644 --- a/cmd/kops-controller/pkg/server/server.go +++ b/cmd/kops-controller/pkg/server/server.go @@ -18,9 +18,15 @@ package server import ( "crypto/tls" + "encoding/json" + "fmt" "net/http" + "runtime/debug" + "github.com/gorilla/mux" + "k8s.io/klog" "k8s.io/kops/cmd/kops-controller/pkg/config" + "k8s.io/kops/pkg/apis/nodeup" ) type Server struct { @@ -36,12 +42,64 @@ func NewServer(opt *config.Options) (*Server, error) { PreferServerCipherSuites: true, }, } - return &Server{ + + s := &Server{ opt: opt, server: server, - }, nil + } + r := mux.NewRouter() + r.Handle("/bootstrap", http.HandlerFunc(s.bootstrap)) + server.Handler = recovery(r) + + return s, nil } func (s *Server) Start() error { return s.server.ListenAndServeTLS(s.opt.Server.ServerCertificatePath, s.opt.Server.ServerKeyPath) } + +func (s *Server) bootstrap(w http.ResponseWriter, r *http.Request) { + if r.Body == nil { + klog.Infof("bootstrap %s no body", r.RemoteAddr) + w.WriteHeader(http.StatusBadRequest) + return + } + + // TODO: authenticate request + + req := &nodeup.BootstrapRequest{} + err := json.NewDecoder(r.Body).Decode(req) + if err != nil { + klog.Infof("bootstrap %s decode err: %v", r.RemoteAddr, err) + w.WriteHeader(http.StatusBadRequest) + _, _ = w.Write([]byte(fmt.Sprintf("failed to decode: %v", err))) + return + } + + if req.APIVersion != nodeup.BootstrapAPIVersion { + klog.Infof("bootstrap %s wrong APIVersion", r.RemoteAddr) + w.WriteHeader(http.StatusBadRequest) + _, _ = w.Write([]byte("unexpected APIVersion")) + return + } + + w.Header().Set("Content-Type", "application/json") + resp := &nodeup.BootstrapResponse{} + _ = json.NewEncoder(w).Encode(resp) + klog.Infof("bootstrap %s success", r.RemoteAddr) +} + +// recovery is responsible for ensuring we don't exit on a panic. +func recovery(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + defer func() { + if err := recover(); err != nil { + w.WriteHeader(http.StatusInternalServerError) + + klog.Errorf("failed to handle request: threw exception: %v: %s", err, debug.Stack()) + } + }() + + next.ServeHTTP(w, req) + }) +} diff --git a/nodeup/pkg/model/BUILD.bazel b/nodeup/pkg/model/BUILD.bazel index ac26b90d5047d..1c393a5055b55 100644 --- a/nodeup/pkg/model/BUILD.bazel +++ b/nodeup/pkg/model/BUILD.bazel @@ -4,6 +4,7 @@ go_library( name = "go_default_library", srcs = [ "architecture.go", + "bootstrap_client.go", "cloudconfig.go", "containerd.go", "context.go", diff --git a/nodeup/pkg/model/bootstrap_client.go b/nodeup/pkg/model/bootstrap_client.go new file mode 100644 index 0000000000000..9ed8c389dc5f5 --- /dev/null +++ b/nodeup/pkg/model/bootstrap_client.go @@ -0,0 +1,46 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package model + +import ( + "k8s.io/kops/upup/pkg/fi" + "k8s.io/kops/upup/pkg/fi/nodeup/nodetasks" +) + +// BootstrapClientBuilder calls kops-controller to bootstrap the node. +type BootstrapClientBuilder struct { + *NodeupModelContext +} + +func (b BootstrapClientBuilder) Build(c *fi.ModelBuilderContext) error { + if b.IsMaster || !b.UseKopsControllerForNodeBootstrap() { + return nil + } + + cert, err := b.GetCert(fi.CertificateIDCA) + if err != nil { + return err + } + + bootstrapClient := &nodetasks.BootstrapClient{ + CA: cert, + } + c.AddTask(bootstrapClient) + return nil +} + +var _ fi.ModelBuilder = &BootstrapClientBuilder{} diff --git a/pkg/apis/nodeup/BUILD.bazel b/pkg/apis/nodeup/BUILD.bazel index 36622381b987b..21643fe611128 100644 --- a/pkg/apis/nodeup/BUILD.bazel +++ b/pkg/apis/nodeup/BUILD.bazel @@ -2,7 +2,10 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library") go_library( name = "go_default_library", - srcs = ["config.go"], + srcs = [ + "bootstrap.go", + "config.go", + ], importpath = "k8s.io/kops/pkg/apis/nodeup", visibility = ["//visibility:public"], deps = [ diff --git a/pkg/apis/nodeup/bootstrap.go b/pkg/apis/nodeup/bootstrap.go new file mode 100644 index 0000000000000..e2b8119b7b761 --- /dev/null +++ b/pkg/apis/nodeup/bootstrap.go @@ -0,0 +1,29 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package nodeup + +const BootstrapAPIVersion = "bootstrap.kops.k8s.io/v1alpha1" + +// BootstrapRequest is a request from nodeup to kops-controller for bootstrapping a node. +type BootstrapRequest struct { + // APIVersion defines the versioned schema of this representation of a request. + APIVersion string `json:"apiVersion"` +} + +// BootstrapRespose is a response to a BootstrapRequest. +type BootstrapResponse struct { +} diff --git a/upup/pkg/fi/nodeup/command.go b/upup/pkg/fi/nodeup/command.go index 0772345fc6166..f75666da7d874 100644 --- a/upup/pkg/fi/nodeup/command.go +++ b/upup/pkg/fi/nodeup/command.go @@ -218,6 +218,7 @@ func (c *NodeUpCommand) Run(out io.Writer) error { } loader := &Loader{} + loader.Builders = append(loader.Builders, &model.BootstrapClientBuilder{NodeupModelContext: modelContext}) loader.Builders = append(loader.Builders, &model.NTPBuilder{NodeupModelContext: modelContext}) loader.Builders = append(loader.Builders, &model.MiscUtilsBuilder{NodeupModelContext: modelContext}) loader.Builders = append(loader.Builders, &model.DirectoryBuilder{NodeupModelContext: modelContext}) diff --git a/upup/pkg/fi/nodeup/nodetasks/BUILD.bazel b/upup/pkg/fi/nodeup/nodetasks/BUILD.bazel index 676e74f8cc7ea..9424293adf214 100644 --- a/upup/pkg/fi/nodeup/nodetasks/BUILD.bazel +++ b/upup/pkg/fi/nodeup/nodetasks/BUILD.bazel @@ -5,6 +5,7 @@ go_library( srcs = [ "archive.go", "bindmount.go", + "bootstrap_client.go", "chattr.go", "createsdir.go", "file.go", @@ -22,9 +23,11 @@ go_library( deps = [ "//nodeup/pkg/distros:go_default_library", "//pkg/apis/kops:go_default_library", + "//pkg/apis/nodeup:go_default_library", "//pkg/backoff:go_default_library", "//pkg/kubeconfig:go_default_library", "//pkg/pki:go_default_library", + "//pkg/wellknownports:go_default_library", "//upup/pkg/fi:go_default_library", "//upup/pkg/fi/nodeup/cloudinit:go_default_library", "//upup/pkg/fi/nodeup/local:go_default_library", diff --git a/upup/pkg/fi/nodeup/nodetasks/bootstrap_client.go b/upup/pkg/fi/nodeup/nodetasks/bootstrap_client.go new file mode 100644 index 0000000000000..17c6a20ceade5 --- /dev/null +++ b/upup/pkg/fi/nodeup/nodetasks/bootstrap_client.go @@ -0,0 +1,125 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package nodetasks + +import ( + "bufio" + "bytes" + "crypto/tls" + "crypto/x509" + "encoding/json" + "fmt" + "io/ioutil" + "net" + "net/http" + "net/url" + "strconv" + + "k8s.io/kops/pkg/apis/nodeup" + "k8s.io/kops/pkg/wellknownports" + "k8s.io/kops/upup/pkg/fi" +) + +type BootstrapClient struct { + // CA is the CA certificate for kops-controller. + CA []byte + + client *http.Client +} + +var _ fi.Task = &BootstrapClient{} +var _ fi.HasName = &BootstrapClient{} + +func (b *BootstrapClient) GetName() *string { + name := "BootstrapClient" + return &name +} + +func (b *BootstrapClient) String() string { + return "BootstrapClient" +} + +func (b *BootstrapClient) Run(c *fi.Context) error { + req := nodeup.BootstrapRequest{ + APIVersion: nodeup.BootstrapAPIVersion, + } + + err := b.queryBootstrap(c, req) + if err != nil { + return err + } + + return nil +} + +func (b *BootstrapClient) queryBootstrap(c *fi.Context, req nodeup.BootstrapRequest) error { + if b.client == nil { + certPool := x509.NewCertPool() + certPool.AppendCertsFromPEM(b.CA) + + b.client = &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{ + RootCAs: certPool, + MinVersion: tls.VersionTLS12, + }, + }, + } + } + + reqBytes, err := json.Marshal(req) + if err != nil { + return err + } + + bootstrapUrl := url.URL{ + Scheme: "https", + Host: net.JoinHostPort(c.Cluster.Spec.MasterInternalName, strconv.Itoa(wellknownports.KopsControllerPort)), + Path: "/bootstrap", + } + resp, err := b.client.Post(bootstrapUrl.String(), "application/json", bytes.NewReader(reqBytes)) + if err != nil { + return err + } + if resp.Body != nil { + defer resp.Body.Close() + } + + if resp.StatusCode != http.StatusOK { + detail := "" + if resp.Body != nil { + scanner := bufio.NewScanner(resp.Body) + if scanner.Scan() { + detail = scanner.Text() + } + } + return fmt.Errorf("bootstrap returned status code %d: %s", resp.StatusCode, detail) + } + + var bootstrapResp nodeup.BootstrapResponse + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return err + } + + err = json.Unmarshal(body, &bootstrapResp) + if err != nil { + return err + } + + return nil +} From cfa262a81a373a236bc36b2535e074185bff4e96 Mon Sep 17 00:00:00 2001 From: John Gardiner Myers Date: Sat, 18 Jul 2020 22:33:09 -0700 Subject: [PATCH 04/13] Authenticate from nodeup to kops-controller --- cmd/kops-controller/BUILD.bazel | 3 + cmd/kops-controller/main.go | 18 ++- cmd/kops-controller/pkg/config/BUILD.bazel | 1 + cmd/kops-controller/pkg/config/options.go | 5 + cmd/kops-controller/pkg/server/BUILD.bazel | 1 + cmd/kops-controller/pkg/server/server.go | 34 +++- nodeup/pkg/model/bootstrap_client.go | 19 ++- upup/pkg/fi/BUILD.bazel | 1 + upup/pkg/fi/authenticate.go | 27 ++++ upup/pkg/fi/cloudup/awsup/BUILD.bazel | 3 + .../pkg/fi/cloudup/awsup/aws_authenticator.go | 62 ++++++++ upup/pkg/fi/cloudup/awsup/aws_verifier.go | 149 ++++++++++++++++++ upup/pkg/fi/cloudup/template_functions.go | 1 + .../fi/nodeup/nodetasks/bootstrap_client.go | 21 ++- 14 files changed, 335 insertions(+), 10 deletions(-) create mode 100644 upup/pkg/fi/authenticate.go create mode 100644 upup/pkg/fi/cloudup/awsup/aws_authenticator.go create mode 100644 upup/pkg/fi/cloudup/awsup/aws_verifier.go diff --git a/cmd/kops-controller/BUILD.bazel b/cmd/kops-controller/BUILD.bazel index a5172a9fd612e..6753aefc72c99 100644 --- a/cmd/kops-controller/BUILD.bazel +++ b/cmd/kops-controller/BUILD.bazel @@ -9,11 +9,14 @@ go_library( "//cmd/kops-controller/controllers:go_default_library", "//cmd/kops-controller/pkg/config:go_default_library", "//cmd/kops-controller/pkg/server:go_default_library", + "//pkg/apis/kops:go_default_library", "//pkg/nodeidentity:go_default_library", "//pkg/nodeidentity/aws:go_default_library", "//pkg/nodeidentity/do:go_default_library", "//pkg/nodeidentity/gce:go_default_library", "//pkg/nodeidentity/openstack:go_default_library", + "//upup/pkg/fi:go_default_library", + "//upup/pkg/fi/cloudup/awsup:go_default_library", "//vendor/k8s.io/api/core/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", "//vendor/k8s.io/client-go/plugin/pkg/client/auth/gcp:go_default_library", diff --git a/cmd/kops-controller/main.go b/cmd/kops-controller/main.go index d2e8f55760077..fe5086ef9bc6f 100644 --- a/cmd/kops-controller/main.go +++ b/cmd/kops-controller/main.go @@ -30,11 +30,14 @@ import ( "k8s.io/kops/cmd/kops-controller/controllers" "k8s.io/kops/cmd/kops-controller/pkg/config" "k8s.io/kops/cmd/kops-controller/pkg/server" + "k8s.io/kops/pkg/apis/kops" "k8s.io/kops/pkg/nodeidentity" nodeidentityaws "k8s.io/kops/pkg/nodeidentity/aws" nodeidentitydo "k8s.io/kops/pkg/nodeidentity/do" nodeidentitygce "k8s.io/kops/pkg/nodeidentity/gce" nodeidentityos "k8s.io/kops/pkg/nodeidentity/openstack" + "k8s.io/kops/upup/pkg/fi" + "k8s.io/kops/upup/pkg/fi/cloudup/awsup" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/yaml" @@ -83,7 +86,20 @@ func main() { ctrl.SetLogger(klogr.New()) if opt.Server != nil { - srv, err := server.NewServer(&opt) + var verifier fi.Verifier + var err error + switch opt.Server.Provider { + case kops.CloudProviderAWS: + verifier, err = awsup.NewAWSVerifier() + if err != nil { + setupLog.Error(err, "unable to create verifier") + os.Exit(1) + } + default: + klog.Fatalf("server for cloud provider %s is not supported", opt.Server.Provider) + } + + srv, err := server.NewServer(&opt, verifier) if err != nil { setupLog.Error(err, "unable to create server") os.Exit(1) diff --git a/cmd/kops-controller/pkg/config/BUILD.bazel b/cmd/kops-controller/pkg/config/BUILD.bazel index 093d96826049e..dd35d7f59abb8 100644 --- a/cmd/kops-controller/pkg/config/BUILD.bazel +++ b/cmd/kops-controller/pkg/config/BUILD.bazel @@ -5,4 +5,5 @@ go_library( srcs = ["options.go"], importpath = "k8s.io/kops/cmd/kops-controller/pkg/config", visibility = ["//visibility:public"], + deps = ["//pkg/apis/kops:go_default_library"], ) diff --git a/cmd/kops-controller/pkg/config/options.go b/cmd/kops-controller/pkg/config/options.go index fa98e935f3175..b2ed9138b3439 100644 --- a/cmd/kops-controller/pkg/config/options.go +++ b/cmd/kops-controller/pkg/config/options.go @@ -16,6 +16,8 @@ limitations under the License. package config +import "k8s.io/kops/pkg/apis/kops" + type Options struct { Cloud string `json:"cloud,omitempty"` ConfigBase string `json:"configBase,omitempty"` @@ -29,6 +31,9 @@ type ServerOptions struct { // Listen is the network endpoint (ip and port) we should listen on. Listen string + // Provider is the cloud provider. + Provider kops.CloudProviderID `json:"provider"` + // ServerKeyPath is the path to our TLS serving private key. ServerKeyPath string `json:"serverKeyPath,omitempty"` // ServerCertificatePath is the path to our TLS serving certificate. diff --git a/cmd/kops-controller/pkg/server/BUILD.bazel b/cmd/kops-controller/pkg/server/BUILD.bazel index 9c65dc3f40cc0..8ba89610856bb 100644 --- a/cmd/kops-controller/pkg/server/BUILD.bazel +++ b/cmd/kops-controller/pkg/server/BUILD.bazel @@ -8,6 +8,7 @@ go_library( deps = [ "//cmd/kops-controller/pkg/config:go_default_library", "//pkg/apis/nodeup:go_default_library", + "//upup/pkg/fi:go_default_library", "//vendor/github.com/gorilla/mux:go_default_library", "//vendor/k8s.io/klog:go_default_library", ], diff --git a/cmd/kops-controller/pkg/server/server.go b/cmd/kops-controller/pkg/server/server.go index ad24a90f9a38c..223ac002591b1 100644 --- a/cmd/kops-controller/pkg/server/server.go +++ b/cmd/kops-controller/pkg/server/server.go @@ -20,6 +20,7 @@ import ( "crypto/tls" "encoding/json" "fmt" + "io/ioutil" "net/http" "runtime/debug" @@ -27,14 +28,16 @@ import ( "k8s.io/klog" "k8s.io/kops/cmd/kops-controller/pkg/config" "k8s.io/kops/pkg/apis/nodeup" + "k8s.io/kops/upup/pkg/fi" ) type Server struct { - opt *config.Options - server *http.Server + opt *config.Options + server *http.Server + verifier fi.Verifier } -func NewServer(opt *config.Options) (*Server, error) { +func NewServer(opt *config.Options, verifier fi.Verifier) (*Server, error) { server := &http.Server{ Addr: opt.Server.Listen, TLSConfig: &tls.Config{ @@ -44,8 +47,9 @@ func NewServer(opt *config.Options) (*Server, error) { } s := &Server{ - opt: opt, - server: server, + opt: opt, + server: server, + verifier: verifier, } r := mux.NewRouter() r.Handle("/bootstrap", http.HandlerFunc(s.bootstrap)) @@ -65,10 +69,26 @@ func (s *Server) bootstrap(w http.ResponseWriter, r *http.Request) { return } - // TODO: authenticate request + body, err := ioutil.ReadAll(r.Body) + if err != nil { + klog.Infof("bootstrap %s read err: %v", r.RemoteAddr, err) + w.WriteHeader(http.StatusBadRequest) + _, _ = w.Write([]byte(fmt.Sprintf("bootstrap %s failed to read body: %v", r.RemoteAddr, err))) + return + } + + id, err := s.verifier.VerifyToken(r.Header.Get("Authorization"), body) + if err != nil { + klog.Infof("bootstrap %s verify err: %v", r.RemoteAddr, err) + w.WriteHeader(http.StatusForbidden) + _, _ = w.Write([]byte(fmt.Sprintf("failed to verify token: %v", err))) + return + } + + klog.Infof("id is %s", id) // todo do something with id req := &nodeup.BootstrapRequest{} - err := json.NewDecoder(r.Body).Decode(req) + err = json.Unmarshal(body, req) if err != nil { klog.Infof("bootstrap %s decode err: %v", r.RemoteAddr, err) w.WriteHeader(http.StatusBadRequest) diff --git a/nodeup/pkg/model/bootstrap_client.go b/nodeup/pkg/model/bootstrap_client.go index 9ed8c389dc5f5..77bd8dce68977 100644 --- a/nodeup/pkg/model/bootstrap_client.go +++ b/nodeup/pkg/model/bootstrap_client.go @@ -17,7 +17,11 @@ limitations under the License. package model import ( + "fmt" + + "k8s.io/kops/pkg/apis/kops" "k8s.io/kops/upup/pkg/fi" + "k8s.io/kops/upup/pkg/fi/cloudup/awsup" "k8s.io/kops/upup/pkg/fi/nodeup/nodetasks" ) @@ -31,13 +35,26 @@ func (b BootstrapClientBuilder) Build(c *fi.ModelBuilderContext) error { return nil } + var authenticator fi.Authenticator + var err error + switch kops.CloudProviderID(b.Cluster.Spec.CloudProvider) { + case kops.CloudProviderAWS: + authenticator, err = awsup.NewAWSAuthenticator() + default: + return fmt.Errorf("unsupported cloud provider %s", b.Cluster.Spec.CloudProvider) + } + if err != nil { + return err + } + cert, err := b.GetCert(fi.CertificateIDCA) if err != nil { return err } bootstrapClient := &nodetasks.BootstrapClient{ - CA: cert, + Authenticator: authenticator, + CA: cert, } c.AddTask(bootstrapClient) return nil diff --git a/upup/pkg/fi/BUILD.bazel b/upup/pkg/fi/BUILD.bazel index 021c74f44ae7e..f8a7e2db0b4ce 100644 --- a/upup/pkg/fi/BUILD.bazel +++ b/upup/pkg/fi/BUILD.bazel @@ -4,6 +4,7 @@ go_library( name = "go_default_library", srcs = [ "assetstore.go", + "authenticate.go", "ca.go", "changes.go", "clientset_castore.go", diff --git a/upup/pkg/fi/authenticate.go b/upup/pkg/fi/authenticate.go new file mode 100644 index 0000000000000..874beb592f6ca --- /dev/null +++ b/upup/pkg/fi/authenticate.go @@ -0,0 +1,27 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package fi + +// Authenticator generates authentication credentials for requests. +type Authenticator interface { + CreateToken(body []byte) (string, error) +} + +// Verifier verifies authentication credentials for requests. +type Verifier interface { + VerifyToken(token string, body []byte) (string, error) +} diff --git a/upup/pkg/fi/cloudup/awsup/BUILD.bazel b/upup/pkg/fi/cloudup/awsup/BUILD.bazel index c5e5d5a811b6c..4d3dca315b77b 100644 --- a/upup/pkg/fi/cloudup/awsup/BUILD.bazel +++ b/upup/pkg/fi/cloudup/awsup/BUILD.bazel @@ -4,8 +4,10 @@ go_library( name = "go_default_library", srcs = [ "aws_apitarget.go", + "aws_authenticator.go", "aws_cloud.go", "aws_utils.go", + "aws_verifier.go", "instancegroups.go", "logging_retryer.go", "machine_types.go", @@ -44,6 +46,7 @@ go_library( "//vendor/github.com/aws/aws-sdk-go/service/iam/iamiface:go_default_library", "//vendor/github.com/aws/aws-sdk-go/service/route53:go_default_library", "//vendor/github.com/aws/aws-sdk-go/service/route53/route53iface:go_default_library", + "//vendor/github.com/aws/aws-sdk-go/service/sts:go_default_library", "//vendor/k8s.io/api/core/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//vendor/k8s.io/klog:go_default_library", diff --git a/upup/pkg/fi/cloudup/awsup/aws_authenticator.go b/upup/pkg/fi/cloudup/awsup/aws_authenticator.go new file mode 100644 index 0000000000000..7f3e1bc7cf5d5 --- /dev/null +++ b/upup/pkg/fi/cloudup/awsup/aws_authenticator.go @@ -0,0 +1,62 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package awsup + +import ( + "crypto/sha256" + "encoding/base64" + "encoding/json" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/sts" + "k8s.io/kops/upup/pkg/fi" +) + +const AWSAuthenticationTokenPrefix = "x-aws-sts " + +type awsAuthenticator struct { + sts *sts.STS +} + +var _ fi.Authenticator = &awsAuthenticator{} + +func NewAWSAuthenticator() (fi.Authenticator, error) { + config := aws.NewConfig().WithCredentialsChainVerboseErrors(true) + sess, err := session.NewSession(config) + if err != nil { + return nil, err + } + return &awsAuthenticator{ + sts: sts.New(sess), + }, nil +} + +func (a awsAuthenticator) CreateToken(body []byte) (string, error) { + sha := sha256.Sum256(body) + + stsRequest, _ := a.sts.GetCallerIdentityRequest(nil) + + stsRequest.HTTPRequest.Header.Add("X-Kops-Request-SHA", base64.RawStdEncoding.EncodeToString(sha[:])) + err := stsRequest.Sign() + if err != nil { + return "", err + } + + headers, _ := json.Marshal(stsRequest.HTTPRequest.Header) + return AWSAuthenticationTokenPrefix + base64.StdEncoding.EncodeToString(headers), nil +} diff --git a/upup/pkg/fi/cloudup/awsup/aws_verifier.go b/upup/pkg/fi/cloudup/awsup/aws_verifier.go new file mode 100644 index 0000000000000..1f7f0815d34ef --- /dev/null +++ b/upup/pkg/fi/cloudup/awsup/aws_verifier.go @@ -0,0 +1,149 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package awsup + +import ( + "bytes" + "crypto/sha256" + "encoding/base64" + "encoding/json" + "encoding/xml" + "fmt" + "io" + "io/ioutil" + "net" + "net/http" + "strconv" + "strings" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/sts" + "k8s.io/kops/upup/pkg/fi" +) + +type awsVerifier struct { + sts *sts.STS + client http.Client +} + +var _ fi.Verifier = &awsVerifier{} + +func NewAWSVerifier() (fi.Verifier, error) { + config := aws.NewConfig().WithCredentialsChainVerboseErrors(true) + sess, err := session.NewSession(config) + if err != nil { + return nil, err + } + return &awsVerifier{ + sts: sts.New(sess), + client: http.Client{ + Transport: &http.Transport{ + Proxy: http.ProxyFromEnvironment, + DialContext: (&net.Dialer{ + Timeout: 30 * time.Second, + }).DialContext, + ForceAttemptHTTP2: true, + DisableKeepAlives: true, + MaxIdleConnsPerHost: -1, + TLSHandshakeTimeout: 10 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + }, + CheckRedirect: func(req *http.Request, via []*http.Request) error { + return http.ErrUseLastResponse + }, + }, + }, nil +} + +type GetCallerIdentityResponse struct { + XMLName xml.Name `xml:"GetCallerIdentityResponse"` + GetCallerIdentityResult []GetCallerIdentityResult `xml:"GetCallerIdentityResult"` + ResponseMetadata []ResponseMetadata `xml:"ResponseMetadata"` +} + +type GetCallerIdentityResult struct { + Arn string `xml:"Arn"` + UserId string `xml:"UserId"` + Account string `xml:"Account"` +} + +type ResponseMetadata struct { + RequestId string `xml:"RequestId"` +} + +func (a awsVerifier) VerifyToken(token string, body []byte) (string, error) { + if !strings.HasPrefix(token, AWSAuthenticationTokenPrefix) { + return "", fmt.Errorf("incorrect authorization type") + } + token = strings.TrimPrefix(token, AWSAuthenticationTokenPrefix) + + // We rely on the client and server using the same version of the same STS library. + stsRequest, _ := a.sts.GetCallerIdentityRequest(nil) + err := stsRequest.Sign() + if err != nil { + return "", fmt.Errorf("creating identity request: %v", err) + } + + stsRequest.HTTPRequest.Header = nil + tokenBytes, err := base64.StdEncoding.DecodeString(token) + if err != nil { + return "", fmt.Errorf("decoding authorization token: %v", err) + } + err = json.Unmarshal(tokenBytes, &stsRequest.HTTPRequest.Header) + if err != nil { + return "", fmt.Errorf("unmarshalling authorization token: %v", err) + } + + sha := sha256.Sum256(body) + if stsRequest.HTTPRequest.Header.Get("X-Kops-Request-SHA") != base64.RawStdEncoding.EncodeToString(sha[:]) { + return "", fmt.Errorf("incorrect SHA") + } + + requestBytes, _ := ioutil.ReadAll(stsRequest.Body) + _, _ = stsRequest.Body.Seek(0, io.SeekStart) + if stsRequest.HTTPRequest.Header.Get("Content-Length") != strconv.Itoa(len(requestBytes)) { + return "", fmt.Errorf("incorrect content-length") + } + + // TODO - implement retry? + response, err := a.client.Do(stsRequest.HTTPRequest) + if err != nil { + return "", fmt.Errorf("sending STS request: %v", err) + } + if response != nil { + defer response.Body.Close() + } + + responseBody, err := ioutil.ReadAll(response.Body) + if err != nil { + return "", fmt.Errorf("reading STS response: %v", err) + } + if response.StatusCode != 200 { + return "", fmt.Errorf("received status code %d from STS: %s", response.StatusCode, string(responseBody)) + } + + result := GetCallerIdentityResponse{} + err = xml.NewDecoder(bytes.NewReader(responseBody)).Decode(&result) + if err != nil { + return "", fmt.Errorf("decoding STS response: %v", err) + } + + marshal, _ := json.Marshal(result) + return string(marshal), nil +} diff --git a/upup/pkg/fi/cloudup/template_functions.go b/upup/pkg/fi/cloudup/template_functions.go index 1430c8299a88f..950d42317bfb1 100644 --- a/upup/pkg/fi/cloudup/template_functions.go +++ b/upup/pkg/fi/cloudup/template_functions.go @@ -384,6 +384,7 @@ func (tf *TemplateFunctions) KopsControllerConfig() (string, error) { pkiDir := "/etc/kubernetes/kops-controller/pki" config.Server = &kopscontrollerconfig.ServerOptions{ Listen: fmt.Sprintf(":%d", wellknownports.KopsControllerPort), + Provider: kops.CloudProviderID(cluster.Spec.CloudProvider), ServerCertificatePath: path.Join(pkiDir, "kops-controller.crt"), ServerKeyPath: path.Join(pkiDir, "kops-controller.key"), } diff --git a/upup/pkg/fi/nodeup/nodetasks/bootstrap_client.go b/upup/pkg/fi/nodeup/nodetasks/bootstrap_client.go index 17c6a20ceade5..5844b572360f8 100644 --- a/upup/pkg/fi/nodeup/nodetasks/bootstrap_client.go +++ b/upup/pkg/fi/nodeup/nodetasks/bootstrap_client.go @@ -35,6 +35,8 @@ import ( ) type BootstrapClient struct { + // Authenticator generates authentication credentials for requests. + Authenticator fi.Authenticator // CA is the CA certificate for kops-controller. CA []byte @@ -43,6 +45,11 @@ type BootstrapClient struct { var _ fi.Task = &BootstrapClient{} var _ fi.HasName = &BootstrapClient{} +var _ fi.HasDependencies = &BootstrapClient{} + +func (b *BootstrapClient) GetDependencies(tasks map[string]fi.Task) []fi.Task { + return nil +} func (b *BootstrapClient) GetName() *string { name := "BootstrapClient" @@ -91,7 +98,19 @@ func (b *BootstrapClient) queryBootstrap(c *fi.Context, req nodeup.BootstrapRequ Host: net.JoinHostPort(c.Cluster.Spec.MasterInternalName, strconv.Itoa(wellknownports.KopsControllerPort)), Path: "/bootstrap", } - resp, err := b.client.Post(bootstrapUrl.String(), "application/json", bytes.NewReader(reqBytes)) + httpReq, err := http.NewRequest("POST", bootstrapUrl.String(), bytes.NewReader(reqBytes)) + if err != nil { + return err + } + httpReq.Header.Set("Content-Type", "application/json") + + token, err := b.Authenticator.CreateToken(reqBytes) + if err != nil { + return err + } + httpReq.Header.Set("Authorization", token) + + resp, err := b.client.Do(httpReq) if err != nil { return err } From 9cfa169740a0da90408c48921c24532c5cbc6513 Mon Sep 17 00:00:00 2001 From: John Gardiner Myers Date: Sat, 1 Aug 2020 17:56:13 -0700 Subject: [PATCH 05/13] Add server code to kops-controller --- cmd/kops-controller/BUILD.bazel | 1 - cmd/kops-controller/main.go | 10 +- cmd/kops-controller/pkg/config/BUILD.bazel | 2 +- cmd/kops-controller/pkg/config/options.go | 8 +- cmd/kops-controller/pkg/server/server.go | 2 +- upup/pkg/fi/authenticate.go | 8 +- upup/pkg/fi/cloudup/awsup/BUILD.bazel | 1 + upup/pkg/fi/cloudup/awsup/aws_verifier.go | 111 ++++++++++++++++++--- upup/pkg/fi/cloudup/template_functions.go | 22 +++- 9 files changed, 137 insertions(+), 28 deletions(-) diff --git a/cmd/kops-controller/BUILD.bazel b/cmd/kops-controller/BUILD.bazel index 6753aefc72c99..6e921b82d0573 100644 --- a/cmd/kops-controller/BUILD.bazel +++ b/cmd/kops-controller/BUILD.bazel @@ -9,7 +9,6 @@ go_library( "//cmd/kops-controller/controllers:go_default_library", "//cmd/kops-controller/pkg/config:go_default_library", "//cmd/kops-controller/pkg/server:go_default_library", - "//pkg/apis/kops:go_default_library", "//pkg/nodeidentity:go_default_library", "//pkg/nodeidentity/aws:go_default_library", "//pkg/nodeidentity/do:go_default_library", diff --git a/cmd/kops-controller/main.go b/cmd/kops-controller/main.go index fe5086ef9bc6f..0ff547323c0d2 100644 --- a/cmd/kops-controller/main.go +++ b/cmd/kops-controller/main.go @@ -30,7 +30,6 @@ import ( "k8s.io/kops/cmd/kops-controller/controllers" "k8s.io/kops/cmd/kops-controller/pkg/config" "k8s.io/kops/cmd/kops-controller/pkg/server" - "k8s.io/kops/pkg/apis/kops" "k8s.io/kops/pkg/nodeidentity" nodeidentityaws "k8s.io/kops/pkg/nodeidentity/aws" nodeidentitydo "k8s.io/kops/pkg/nodeidentity/do" @@ -88,15 +87,14 @@ func main() { if opt.Server != nil { var verifier fi.Verifier var err error - switch opt.Server.Provider { - case kops.CloudProviderAWS: - verifier, err = awsup.NewAWSVerifier() + if opt.Server.Provider.AWS != nil { + verifier, err = awsup.NewAWSVerifier(opt.Server.Provider.AWS) if err != nil { setupLog.Error(err, "unable to create verifier") os.Exit(1) } - default: - klog.Fatalf("server for cloud provider %s is not supported", opt.Server.Provider) + } else { + klog.Fatalf("server cloud provider config not provided") } srv, err := server.NewServer(&opt, verifier) diff --git a/cmd/kops-controller/pkg/config/BUILD.bazel b/cmd/kops-controller/pkg/config/BUILD.bazel index dd35d7f59abb8..ee82bdbfb7d19 100644 --- a/cmd/kops-controller/pkg/config/BUILD.bazel +++ b/cmd/kops-controller/pkg/config/BUILD.bazel @@ -5,5 +5,5 @@ go_library( srcs = ["options.go"], importpath = "k8s.io/kops/cmd/kops-controller/pkg/config", visibility = ["//visibility:public"], - deps = ["//pkg/apis/kops:go_default_library"], + deps = ["//upup/pkg/fi/cloudup/awsup:go_default_library"], ) diff --git a/cmd/kops-controller/pkg/config/options.go b/cmd/kops-controller/pkg/config/options.go index b2ed9138b3439..cdb2b4d0e3177 100644 --- a/cmd/kops-controller/pkg/config/options.go +++ b/cmd/kops-controller/pkg/config/options.go @@ -16,7 +16,7 @@ limitations under the License. package config -import "k8s.io/kops/pkg/apis/kops" +import "k8s.io/kops/upup/pkg/fi/cloudup/awsup" type Options struct { Cloud string `json:"cloud,omitempty"` @@ -32,10 +32,14 @@ type ServerOptions struct { Listen string // Provider is the cloud provider. - Provider kops.CloudProviderID `json:"provider"` + Provider ServerProviderOptions `json:"provider"` // ServerKeyPath is the path to our TLS serving private key. ServerKeyPath string `json:"serverKeyPath,omitempty"` // ServerCertificatePath is the path to our TLS serving certificate. ServerCertificatePath string `json:"serverCertificatePath,omitempty"` } + +type ServerProviderOptions struct { + AWS *awsup.AWSVerifierOptions `json:"aws,omitempty"` +} diff --git a/cmd/kops-controller/pkg/server/server.go b/cmd/kops-controller/pkg/server/server.go index 223ac002591b1..0210d5b96ed57 100644 --- a/cmd/kops-controller/pkg/server/server.go +++ b/cmd/kops-controller/pkg/server/server.go @@ -85,7 +85,7 @@ func (s *Server) bootstrap(w http.ResponseWriter, r *http.Request) { return } - klog.Infof("id is %s", id) // todo do something with id + klog.Infof("id is %s", id.Instance) // todo do something with id req := &nodeup.BootstrapRequest{} err = json.Unmarshal(body, req) diff --git a/upup/pkg/fi/authenticate.go b/upup/pkg/fi/authenticate.go index 874beb592f6ca..2e96bac4e252b 100644 --- a/upup/pkg/fi/authenticate.go +++ b/upup/pkg/fi/authenticate.go @@ -21,7 +21,13 @@ type Authenticator interface { CreateToken(body []byte) (string, error) } +// VerifyResult is the result of a successfully verified request. +type VerifyResult struct { + // Instance is the hostname of the instance. + Instance string +} + // Verifier verifies authentication credentials for requests. type Verifier interface { - VerifyToken(token string, body []byte) (string, error) + VerifyToken(token string, body []byte) (*VerifyResult, error) } diff --git a/upup/pkg/fi/cloudup/awsup/BUILD.bazel b/upup/pkg/fi/cloudup/awsup/BUILD.bazel index 4d3dca315b77b..193241329503a 100644 --- a/upup/pkg/fi/cloudup/awsup/BUILD.bazel +++ b/upup/pkg/fi/cloudup/awsup/BUILD.bazel @@ -30,6 +30,7 @@ go_library( "//vendor/github.com/aws/aws-sdk-go/aws:go_default_library", "//vendor/github.com/aws/aws-sdk-go/aws/awserr:go_default_library", "//vendor/github.com/aws/aws-sdk-go/aws/client:go_default_library", + "//vendor/github.com/aws/aws-sdk-go/aws/ec2metadata:go_default_library", "//vendor/github.com/aws/aws-sdk-go/aws/endpoints:go_default_library", "//vendor/github.com/aws/aws-sdk-go/aws/request:go_default_library", "//vendor/github.com/aws/aws-sdk-go/aws/session:go_default_library", diff --git a/upup/pkg/fi/cloudup/awsup/aws_verifier.go b/upup/pkg/fi/cloudup/awsup/aws_verifier.go index 1f7f0815d34ef..b19852a40fa9c 100644 --- a/upup/pkg/fi/cloudup/awsup/aws_verifier.go +++ b/upup/pkg/fi/cloudup/awsup/aws_verifier.go @@ -32,26 +32,55 @@ import ( "time" "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/ec2metadata" "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/ec2" "github.com/aws/aws-sdk-go/service/sts" "k8s.io/kops/upup/pkg/fi" ) +type AWSVerifierOptions struct { + // NodesRoles are the IAM roles that worker nodes are permitted to have. + NodesRoles []string `json:"nodesRoles"` +} + type awsVerifier struct { + accountId string + opt AWSVerifierOptions + + ec2 *ec2.EC2 sts *sts.STS client http.Client } var _ fi.Verifier = &awsVerifier{} -func NewAWSVerifier() (fi.Verifier, error) { +func NewAWSVerifier(opt *AWSVerifierOptions) (fi.Verifier, error) { config := aws.NewConfig().WithCredentialsChainVerboseErrors(true) sess, err := session.NewSession(config) if err != nil { return nil, err } + + stsClient := sts.New(sess) + identity, err := stsClient.GetCallerIdentity(&sts.GetCallerIdentityInput{}) + if err != nil { + return nil, err + } + + metadata := ec2metadata.New(sess, config) + region, err := metadata.Region() + if err != nil { + return nil, fmt.Errorf("error querying ec2 metadata service (for region): %v", err) + } + + ec2Client := ec2.New(sess, config.WithRegion(region)) + return &awsVerifier{ - sts: sts.New(sess), + accountId: aws.StringValue(identity.Account), + opt: *opt, + ec2: ec2Client, + sts: stsClient, client: http.Client{ Transport: &http.Transport{ Proxy: http.ProxyFromEnvironment, @@ -87,9 +116,9 @@ type ResponseMetadata struct { RequestId string `xml:"RequestId"` } -func (a awsVerifier) VerifyToken(token string, body []byte) (string, error) { +func (a awsVerifier) VerifyToken(token string, body []byte) (*fi.VerifyResult, error) { if !strings.HasPrefix(token, AWSAuthenticationTokenPrefix) { - return "", fmt.Errorf("incorrect authorization type") + return nil, fmt.Errorf("incorrect authorization type") } token = strings.TrimPrefix(token, AWSAuthenticationTokenPrefix) @@ -97,34 +126,34 @@ func (a awsVerifier) VerifyToken(token string, body []byte) (string, error) { stsRequest, _ := a.sts.GetCallerIdentityRequest(nil) err := stsRequest.Sign() if err != nil { - return "", fmt.Errorf("creating identity request: %v", err) + return nil, fmt.Errorf("creating identity request: %v", err) } stsRequest.HTTPRequest.Header = nil tokenBytes, err := base64.StdEncoding.DecodeString(token) if err != nil { - return "", fmt.Errorf("decoding authorization token: %v", err) + return nil, fmt.Errorf("decoding authorization token: %v", err) } err = json.Unmarshal(tokenBytes, &stsRequest.HTTPRequest.Header) if err != nil { - return "", fmt.Errorf("unmarshalling authorization token: %v", err) + return nil, fmt.Errorf("unmarshalling authorization token: %v", err) } sha := sha256.Sum256(body) if stsRequest.HTTPRequest.Header.Get("X-Kops-Request-SHA") != base64.RawStdEncoding.EncodeToString(sha[:]) { - return "", fmt.Errorf("incorrect SHA") + return nil, fmt.Errorf("incorrect SHA") } requestBytes, _ := ioutil.ReadAll(stsRequest.Body) _, _ = stsRequest.Body.Seek(0, io.SeekStart) if stsRequest.HTTPRequest.Header.Get("Content-Length") != strconv.Itoa(len(requestBytes)) { - return "", fmt.Errorf("incorrect content-length") + return nil, fmt.Errorf("incorrect content-length") } // TODO - implement retry? response, err := a.client.Do(stsRequest.HTTPRequest) if err != nil { - return "", fmt.Errorf("sending STS request: %v", err) + return nil, fmt.Errorf("sending STS request: %v", err) } if response != nil { defer response.Body.Close() @@ -132,18 +161,70 @@ func (a awsVerifier) VerifyToken(token string, body []byte) (string, error) { responseBody, err := ioutil.ReadAll(response.Body) if err != nil { - return "", fmt.Errorf("reading STS response: %v", err) + return nil, fmt.Errorf("reading STS response: %v", err) } if response.StatusCode != 200 { - return "", fmt.Errorf("received status code %d from STS: %s", response.StatusCode, string(responseBody)) + return nil, fmt.Errorf("received status code %d from STS: %s", response.StatusCode, string(responseBody)) } result := GetCallerIdentityResponse{} err = xml.NewDecoder(bytes.NewReader(responseBody)).Decode(&result) if err != nil { - return "", fmt.Errorf("decoding STS response: %v", err) + return nil, fmt.Errorf("decoding STS response: %v", err) } - marshal, _ := json.Marshal(result) - return string(marshal), nil + if result.GetCallerIdentityResult[0].Account != a.accountId { + return nil, fmt.Errorf("incorrect account %s", result.GetCallerIdentityResult[0].Account) + } + + arn := result.GetCallerIdentityResult[0].Arn + parts := strings.Split(arn, ":") + if len(parts) != 6 { + return nil, fmt.Errorf("arn %q contains unexpected number of colons", arn) + } + if parts[0] != "arn" { + return nil, fmt.Errorf("arn %q doesn't start with \"arn:\"", arn) + } + // parts[1] is partition + if parts[2] != "iam" && parts[2] != "sts" { + return nil, fmt.Errorf("arn %q has unrecognized service", arn) + } + // parts[3] is region + // parts[4] is account + resource := strings.Split(parts[5], "/") + if resource[0] != "assumed-role" { + return nil, fmt.Errorf("arn %q has unrecognized type", arn) + } + if len(resource) < 3 { + return nil, fmt.Errorf("arn %q contains too few slashes", arn) + } + found := false + for _, role := range a.opt.NodesRoles { + if resource[1] == role { + found = true + break + } + } + if !found { + return nil, fmt.Errorf("arn %q does not contain acceptable node role", arn) + } + + instanceID := resource[2] + instances, err := a.ec2.DescribeInstances(&ec2.DescribeInstancesInput{ + InstanceIds: aws.StringSlice([]string{instanceID}), + }) + if err != nil { + return nil, fmt.Errorf("describing instance for arn %q", arn) + } + + if len(instances.Reservations) <= 0 || len(instances.Reservations[0].Instances) <= 0 { + return nil, fmt.Errorf("missing instance id: %s", instanceID) + } + if len(instances.Reservations[0].Instances) > 1 { + return nil, fmt.Errorf("found multiple instances with instance id: %s", instanceID) + } + + return &fi.VerifyResult{ + Instance: aws.StringValue(instances.Reservations[0].Instances[0].PrivateDnsName), + }, nil } diff --git a/upup/pkg/fi/cloudup/template_functions.go b/upup/pkg/fi/cloudup/template_functions.go index 950d42317bfb1..bb598db88b24d 100644 --- a/upup/pkg/fi/cloudup/template_functions.go +++ b/upup/pkg/fi/cloudup/template_functions.go @@ -38,6 +38,7 @@ import ( "github.com/Masterminds/sprig" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/sets" "k8s.io/klog" kopscontrollerconfig "k8s.io/kops/cmd/kops-controller/pkg/config" "k8s.io/kops/pkg/apis/kops" @@ -48,6 +49,7 @@ import ( "k8s.io/kops/pkg/resources/spotinst" "k8s.io/kops/pkg/wellknownports" "k8s.io/kops/upup/pkg/fi" + "k8s.io/kops/upup/pkg/fi/cloudup/awsup" "k8s.io/kops/upup/pkg/fi/cloudup/gce" "k8s.io/kops/util/pkg/env" ) @@ -384,10 +386,28 @@ func (tf *TemplateFunctions) KopsControllerConfig() (string, error) { pkiDir := "/etc/kubernetes/kops-controller/pki" config.Server = &kopscontrollerconfig.ServerOptions{ Listen: fmt.Sprintf(":%d", wellknownports.KopsControllerPort), - Provider: kops.CloudProviderID(cluster.Spec.CloudProvider), ServerCertificatePath: path.Join(pkiDir, "kops-controller.crt"), ServerKeyPath: path.Join(pkiDir, "kops-controller.key"), } + + switch kops.CloudProviderID(cluster.Spec.CloudProvider) { + case kops.CloudProviderAWS: + nodesRoles := sets.String{} + for _, ig := range tf.InstanceGroups { + if ig.Spec.Role == kops.InstanceGroupRoleNode { + profile, err := tf.LinkToIAMInstanceProfile(ig) + if err != nil { + return "", fmt.Errorf("getting role for ig %s: %v", ig.Name, err) + } + nodesRoles.Insert(*profile.Name) + } + } + config.Server.Provider.AWS = &awsup.AWSVerifierOptions{ + NodesRoles: nodesRoles.List(), + } + default: + return "", fmt.Errorf("unsupported cloud provider %s", cluster.Spec.CloudProvider) + } } // To avoid indentation problems, we marshal as json. json is a subset of yaml From 321035f46008feed980389028f3e3f6aea55c8aa Mon Sep 17 00:00:00 2001 From: John Gardiner Myers Date: Sun, 2 Aug 2020 16:55:31 -0700 Subject: [PATCH 06/13] Allow cert/key file tasks to specify owner --- nodeup/pkg/model/context.go | 18 ++++++++++-------- nodeup/pkg/model/etcd_manager_tls.go | 4 ++-- nodeup/pkg/model/kube_controller_manager.go | 2 +- nodeup/pkg/model/networking/calico.go | 6 +++--- nodeup/pkg/model/networking/cilium.go | 4 ++-- nodeup/pkg/model/node_authorizer.go | 8 ++++---- nodeup/pkg/model/protokube.go | 4 ++-- nodeup/pkg/model/secrets.go | 4 ++-- 8 files changed, 26 insertions(+), 24 deletions(-) diff --git a/nodeup/pkg/model/context.go b/nodeup/pkg/model/context.go index 5cd801792b928..fceb1ec8a1a5e 100644 --- a/nodeup/pkg/model/context.go +++ b/nodeup/pkg/model/context.go @@ -378,20 +378,20 @@ func (c *NodeupModelContext) KubectlPath() string { return kubeletCommand } -// BuildCertificatePairTask creates the tasks to pull down the certificate and private key -func (c *NodeupModelContext) BuildCertificatePairTask(ctx *fi.ModelBuilderContext, key, path, filename string) error { +// BuildCertificatePairTask creates the tasks to create the certificate and private key files. +func (c *NodeupModelContext) BuildCertificatePairTask(ctx *fi.ModelBuilderContext, key, path, filename string, owner *string) error { certificateName := filepath.Join(path, filename+".pem") keyName := filepath.Join(path, filename+"-key.pem") - if err := c.BuildCertificateTask(ctx, key, certificateName); err != nil { + if err := c.BuildCertificateTask(ctx, key, certificateName, owner); err != nil { return err } - return c.BuildPrivateKeyTask(ctx, key, keyName) + return c.BuildPrivateKeyTask(ctx, key, keyName, owner) } -// BuildCertificateTask is responsible for build a certificate request task -func (c *NodeupModelContext) BuildCertificateTask(ctx *fi.ModelBuilderContext, name, filename string) error { +// BuildCertificateTask builds a task to create a certificate file. +func (c *NodeupModelContext) BuildCertificateTask(ctx *fi.ModelBuilderContext, name, filename string, owner *string) error { cert, err := c.KeyStore.FindCert(name) if err != nil { return err @@ -416,13 +416,14 @@ func (c *NodeupModelContext) BuildCertificateTask(ctx *fi.ModelBuilderContext, n Contents: fi.NewStringResource(serialized), Type: nodetasks.FileType_File, Mode: s("0600"), + Owner: owner, }) return nil } -// BuildPrivateKeyTask is responsible for build a certificate request task -func (c *NodeupModelContext) BuildPrivateKeyTask(ctx *fi.ModelBuilderContext, name, filename string) error { +// BuildPrivateKeyTask builds a task to create a private key file. +func (c *NodeupModelContext) BuildPrivateKeyTask(ctx *fi.ModelBuilderContext, name, filename string, owner *string) error { cert, err := c.KeyStore.FindPrivateKey(name) if err != nil { return err @@ -447,6 +448,7 @@ func (c *NodeupModelContext) BuildPrivateKeyTask(ctx *fi.ModelBuilderContext, na Contents: fi.NewStringResource(serialized), Type: nodetasks.FileType_File, Mode: s("0600"), + Owner: owner, }) return nil diff --git a/nodeup/pkg/model/etcd_manager_tls.go b/nodeup/pkg/model/etcd_manager_tls.go index 269334334f6e1..1388281a4001e 100644 --- a/nodeup/pkg/model/etcd_manager_tls.go +++ b/nodeup/pkg/model/etcd_manager_tls.go @@ -54,10 +54,10 @@ func (b *EtcdManagerTLSBuilder) Build(ctx *fi.ModelBuilderContext) error { return fmt.Errorf("keypair %q not found", keystoreName) } - if err := b.BuildCertificateTask(ctx, keystoreName, d+"/"+fileName+".crt"); err != nil { + if err := b.BuildCertificateTask(ctx, keystoreName, d+"/"+fileName+".crt", nil); err != nil { return err } - if err := b.BuildPrivateKeyTask(ctx, keystoreName, d+"/"+fileName+".key"); err != nil { + if err := b.BuildPrivateKeyTask(ctx, keystoreName, d+"/"+fileName+".key", nil); err != nil { return err } } diff --git a/nodeup/pkg/model/kube_controller_manager.go b/nodeup/pkg/model/kube_controller_manager.go index 174084bfb7dcc..d61b3274715f0 100644 --- a/nodeup/pkg/model/kube_controller_manager.go +++ b/nodeup/pkg/model/kube_controller_manager.go @@ -53,7 +53,7 @@ func (b *KubeControllerManagerBuilder) Build(c *fi.ModelBuilderContext) error { // Include the CA Key // @TODO: use a per-machine key? use KMS? - if err := b.BuildPrivateKeyTask(c, fi.CertificateIDCA, "ca.key"); err != nil { + if err := b.BuildPrivateKeyTask(c, fi.CertificateIDCA, "ca.key", nil); err != nil { return err } diff --git a/nodeup/pkg/model/networking/calico.go b/nodeup/pkg/model/networking/calico.go index f50e67e9696f4..1f682fd540716 100644 --- a/nodeup/pkg/model/networking/calico.go +++ b/nodeup/pkg/model/networking/calico.go @@ -46,13 +46,13 @@ func (b *CalicoBuilder) Build(c *fi.ModelBuilderContext) error { certificate := filepath.Join(dirname, name+".pem") key := filepath.Join(dirname, name+"-key.pem") - if err := b.BuildCertificateTask(c, name, certificate); err != nil { + if err := b.BuildCertificateTask(c, name, certificate, nil); err != nil { return err } - if err := b.BuildPrivateKeyTask(c, name, key); err != nil { + if err := b.BuildPrivateKeyTask(c, name, key, nil); err != nil { return err } - if err := b.BuildCertificateTask(c, fi.CertificateIDCA, ca); err != nil { + if err := b.BuildCertificateTask(c, fi.CertificateIDCA, ca, nil); err != nil { return err } } diff --git a/nodeup/pkg/model/networking/cilium.go b/nodeup/pkg/model/networking/cilium.go index c3c243d651107..a2585732059f5 100644 --- a/nodeup/pkg/model/networking/cilium.go +++ b/nodeup/pkg/model/networking/cilium.go @@ -123,10 +123,10 @@ func (b *CiliumBuilder) buildCiliumEtcdSecrets(c *fi.ModelBuilderContext) error return err } - if err := b.BuildCertificateTask(c, keystoreName, d+"/"+fileName+".crt"); err != nil { + if err := b.BuildCertificateTask(c, keystoreName, d+"/"+fileName+".crt", nil); err != nil { return err } - if err := b.BuildPrivateKeyTask(c, keystoreName, d+"/"+fileName+".key"); err != nil { + if err := b.BuildPrivateKeyTask(c, keystoreName, d+"/"+fileName+".key", nil); err != nil { return err } } diff --git a/nodeup/pkg/model/node_authorizer.go b/nodeup/pkg/model/node_authorizer.go index e729c8e1bda9e..a3fdfa546a501 100644 --- a/nodeup/pkg/model/node_authorizer.go +++ b/nodeup/pkg/model/node_authorizer.go @@ -43,11 +43,11 @@ func (b *NodeAuthorizationBuilder) Build(c *fi.ModelBuilderContext) error { if b.UseBootstrapTokens() && b.IsMaster { name := "node-authorizer" // creates /src/kubernetes/node-authorizer/{tls,tls-key}.pem - if err := b.BuildCertificatePairTask(c, name, name, "tls"); err != nil { + if err := b.BuildCertificatePairTask(c, name, name, "tls", nil); err != nil { return err } // creates /src/kubernetes/node-authorizer/ca.pem - if err := b.BuildCertificateTask(c, fi.CertificateIDCA, filepath.Join(name, "ca.pem")); err != nil { + if err := b.BuildCertificateTask(c, fi.CertificateIDCA, filepath.Join(name, "ca.pem"), nil); err != nil { return err } } @@ -55,10 +55,10 @@ func (b *NodeAuthorizationBuilder) Build(c *fi.ModelBuilderContext) error { authorizerDir := "node-authorizer" // @check if bootstrap tokens are enabled and download client certificates for nodes if b.UseBootstrapTokens() && !b.IsMaster { - if err := b.BuildCertificatePairTask(c, "node-authorizer-client", authorizerDir, "tls"); err != nil { + if err := b.BuildCertificatePairTask(c, "node-authorizer-client", authorizerDir, "tls", nil); err != nil { return err } - if err := b.BuildCertificateTask(c, fi.CertificateIDCA, authorizerDir+"/ca.pem"); err != nil { + if err := b.BuildCertificateTask(c, fi.CertificateIDCA, authorizerDir+"/ca.pem", nil); err != nil { return err } } diff --git a/nodeup/pkg/model/protokube.go b/nodeup/pkg/model/protokube.go index 1a605c49ae479..ff6daac4fbe04 100644 --- a/nodeup/pkg/model/protokube.go +++ b/nodeup/pkg/model/protokube.go @@ -82,12 +82,12 @@ func (t *ProtokubeBuilder) Build(c *fi.ModelBuilderContext) error { // retrieve the etcd peer certificates and private keys from the keystore if !t.UseEtcdManager() && t.UseEtcdTLS() { for _, x := range []string{"etcd", "etcd-peer", "etcd-client"} { - if err := t.BuildCertificateTask(c, x, fmt.Sprintf("%s.pem", x)); err != nil { + if err := t.BuildCertificateTask(c, x, fmt.Sprintf("%s.pem", x), nil); err != nil { return err } } for _, x := range []string{"etcd", "etcd-peer", "etcd-client"} { - if err := t.BuildPrivateKeyTask(c, x, fmt.Sprintf("%s-key.pem", x)); err != nil { + if err := t.BuildPrivateKeyTask(c, x, fmt.Sprintf("%s-key.pem", x), nil); err != nil { return err } } diff --git a/nodeup/pkg/model/secrets.go b/nodeup/pkg/model/secrets.go index 90985ce379033..7ed566814a8b5 100644 --- a/nodeup/pkg/model/secrets.go +++ b/nodeup/pkg/model/secrets.go @@ -50,7 +50,7 @@ func (b *SecretBuilder) Build(c *fi.ModelBuilderContext) error { } // @step: retrieve the platform ca - if err := b.BuildCertificateTask(c, fi.CertificateIDCA, "ca.crt"); err != nil { + if err := b.BuildCertificateTask(c, fi.CertificateIDCA, "ca.crt", nil); err != nil { return err } @@ -142,7 +142,7 @@ func (b *SecretBuilder) Build(c *fi.ModelBuilderContext) error { } } - if err := b.BuildPrivateKeyTask(c, "master", "service-account.key"); err != nil { + if err := b.BuildPrivateKeyTask(c, "master", "service-account.key", nil); err != nil { return err } From bec273ebf1dec1f8db4eaabf30051a099a160c9a Mon Sep 17 00:00:00 2001 From: John Gardiner Myers Date: Sun, 2 Aug 2020 14:36:55 -0700 Subject: [PATCH 07/13] Implement signing of kubelet cert in kops-controller --- cmd/kops-controller/pkg/config/options.go | 5 ++ cmd/kops-controller/pkg/server/BUILD.bazel | 7 +- cmd/kops-controller/pkg/server/keystore.go | 76 ++++++++++++++++++++++ cmd/kops-controller/pkg/server/server.go | 76 ++++++++++++++++++++-- nodeup/pkg/model/kops_controller.go | 8 +++ pkg/apis/nodeup/bootstrap.go | 4 ++ pkg/pki/issue.go | 9 ++- upup/pkg/fi/cloudup/template_functions.go | 2 + 8 files changed, 180 insertions(+), 7 deletions(-) create mode 100644 cmd/kops-controller/pkg/server/keystore.go diff --git a/cmd/kops-controller/pkg/config/options.go b/cmd/kops-controller/pkg/config/options.go index cdb2b4d0e3177..5a82a3bfbe3e1 100644 --- a/cmd/kops-controller/pkg/config/options.go +++ b/cmd/kops-controller/pkg/config/options.go @@ -38,6 +38,11 @@ type ServerOptions struct { ServerKeyPath string `json:"serverKeyPath,omitempty"` // ServerCertificatePath is the path to our TLS serving certificate. ServerCertificatePath string `json:"serverCertificatePath,omitempty"` + + // CABasePath is a base of the path to CA certificate and key files. + CABasePath string `json:"caBasePath"` + // SigningCAs is the list of active signing CAs. + SigningCAs []string `json:"signingCAs"` } type ServerProviderOptions struct { diff --git a/cmd/kops-controller/pkg/server/BUILD.bazel b/cmd/kops-controller/pkg/server/BUILD.bazel index 8ba89610856bb..522bbaba93d77 100644 --- a/cmd/kops-controller/pkg/server/BUILD.bazel +++ b/cmd/kops-controller/pkg/server/BUILD.bazel @@ -2,12 +2,17 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library") go_library( name = "go_default_library", - srcs = ["server.go"], + srcs = [ + "keystore.go", + "server.go", + ], importpath = "k8s.io/kops/cmd/kops-controller/pkg/server", visibility = ["//visibility:public"], deps = [ "//cmd/kops-controller/pkg/config:go_default_library", "//pkg/apis/nodeup:go_default_library", + "//pkg/pki:go_default_library", + "//pkg/rbac:go_default_library", "//upup/pkg/fi:go_default_library", "//vendor/github.com/gorilla/mux:go_default_library", "//vendor/k8s.io/klog:go_default_library", diff --git a/cmd/kops-controller/pkg/server/keystore.go b/cmd/kops-controller/pkg/server/keystore.go new file mode 100644 index 0000000000000..52dfb8a52bea8 --- /dev/null +++ b/cmd/kops-controller/pkg/server/keystore.go @@ -0,0 +1,76 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package server + +import ( + "fmt" + "io/ioutil" + "path" + + "k8s.io/kops/pkg/pki" +) + +type keystore struct { + keys map[string]keystoreEntry +} + +type keystoreEntry struct { + certificate *pki.Certificate + key *pki.PrivateKey +} + +var _ pki.Keystore = keystore{} + +func (k keystore) FindKeypair(name string) (*pki.Certificate, *pki.PrivateKey, bool, error) { + entry, ok := k.keys[name] + if !ok { + return nil, nil, false, fmt.Errorf("unknown CA %q", name) + } + return entry.certificate, entry.key, false, nil +} + +func newKeystore(basePath string, cas []string) (pki.Keystore, error) { + keystore := &keystore{ + keys: map[string]keystoreEntry{}, + } + for _, name := range cas { + certBytes, err := ioutil.ReadFile(path.Join(basePath, name+".pem")) + if err != nil { + return nil, fmt.Errorf("reading %q certificate: %v", name, err) + } + certificate, err := pki.ParsePEMCertificate(certBytes) + if err != nil { + return nil, fmt.Errorf("parsing %q certificate: %v", name, err) + } + + keyBytes, err := ioutil.ReadFile(path.Join(basePath, name+"-key.pem")) + if err != nil { + return nil, fmt.Errorf("reading %q key: %v", name, err) + } + key, err := pki.ParsePEMPrivateKey(keyBytes) + if err != nil { + return nil, fmt.Errorf("parsing %q key: %v", name, err) + } + + keystore.keys[name] = keystoreEntry{ + certificate: certificate, + key: key, + } + } + + return keystore, nil +} diff --git a/cmd/kops-controller/pkg/server/server.go b/cmd/kops-controller/pkg/server/server.go index 0210d5b96ed57..7a28de8ac3baa 100644 --- a/cmd/kops-controller/pkg/server/server.go +++ b/cmd/kops-controller/pkg/server/server.go @@ -18,16 +18,23 @@ package server import ( "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" "encoding/json" + "encoding/pem" "fmt" + "hash/fnv" "io/ioutil" "net/http" "runtime/debug" + "time" "github.com/gorilla/mux" "k8s.io/klog" "k8s.io/kops/cmd/kops-controller/pkg/config" "k8s.io/kops/pkg/apis/nodeup" + "k8s.io/kops/pkg/pki" + "k8s.io/kops/pkg/rbac" "k8s.io/kops/upup/pkg/fi" ) @@ -35,6 +42,7 @@ type Server struct { opt *config.Options server *http.Server verifier fi.Verifier + keystore pki.Keystore } func NewServer(opt *config.Options, verifier fi.Verifier) (*Server, error) { @@ -59,6 +67,12 @@ func NewServer(opt *config.Options, verifier fi.Verifier) (*Server, error) { } func (s *Server) Start() error { + var err error + s.keystore, err = newKeystore(s.opt.Server.CABasePath, s.opt.Server.SigningCAs) + if err != nil { + return err + } + return s.server.ListenAndServeTLS(s.opt.Server.ServerCertificatePath, s.opt.Server.ServerKeyPath) } @@ -85,8 +99,6 @@ func (s *Server) bootstrap(w http.ResponseWriter, r *http.Request) { return } - klog.Infof("id is %s", id.Instance) // todo do something with id - req := &nodeup.BootstrapRequest{} err = json.Unmarshal(body, req) if err != nil { @@ -103,10 +115,66 @@ func (s *Server) bootstrap(w http.ResponseWriter, r *http.Request) { return } + resp := &nodeup.BootstrapResponse{ + Certs: map[string]string{}, + } + + // Skew the certificate lifetime by up to 30 days based on information about the requesting node. + // This is so that different nodes created at the same time have the certificates they generated + // expire at different times, but all certificates on a given node expire around the same time. + hash := fnv.New32() + _, _ = hash.Write([]byte(r.RemoteAddr)) + validHours := (455 * 24) + (hash.Sum32() % (30 * 24)) + + for name, pubKey := range req.Certs { + cert, err := s.issueCert(name, pubKey, id, validHours) + if err != nil { + klog.Infof("bootstrap %s cert %q issue err: %v", r.RemoteAddr, name, err) + w.WriteHeader(http.StatusBadRequest) + _, _ = w.Write([]byte(fmt.Sprintf("failed to issue %q: %v", name, err))) + return + } + resp.Certs[name] = cert + } + w.Header().Set("Content-Type", "application/json") - resp := &nodeup.BootstrapResponse{} _ = json.NewEncoder(w).Encode(resp) - klog.Infof("bootstrap %s success", r.RemoteAddr) + klog.Infof("bootstrap %s %s success", r.RemoteAddr, id.Instance) +} + +func (s *Server) issueCert(name string, pubKey string, id *fi.VerifyResult, validHours uint32) (string, error) { + block, _ := pem.Decode([]byte(pubKey)) + if block.Type != "RSA PUBLIC KEY" { + return "", fmt.Errorf("unexpected key type %q", block.Type) + } + key, err := x509.ParsePKIXPublicKey(block.Bytes) + if err != nil { + return "", fmt.Errorf("parsing key: %v", err) + } + + issueReq := &pki.IssueCertRequest{ + Signer: fi.CertificateIDCA, + Type: "client", + PublicKey: key, + Validity: time.Hour * time.Duration(validHours), + } + + switch name { + case "kubelet": + issueReq.Subject = pkix.Name{ + CommonName: fmt.Sprintf("system:node:%s", id.Instance), + Organization: []string{rbac.NodesGroup}, + } + default: + return "", fmt.Errorf("unexpected key name") + } + + cert, _, _, err := pki.IssueCert(issueReq, s.keystore) + if err != nil { + return "", fmt.Errorf("issuing certificate: %v", err) + } + + return cert.AsString() } // recovery is responsible for ensuring we don't exit on a panic. diff --git a/nodeup/pkg/model/kops_controller.go b/nodeup/pkg/model/kops_controller.go index 1a27af5e85b5e..d8c6f4789eee0 100644 --- a/nodeup/pkg/model/kops_controller.go +++ b/nodeup/pkg/model/kops_controller.go @@ -81,5 +81,13 @@ func (b *KopsControllerBuilder) Build(c *fi.ModelBuilderContext) error { Owner: s(wellknownusers.KopsControllerName), }) + for _, cert := range []string{fi.CertificateIDCA} { + owner := wellknownusers.KopsControllerName + err := b.BuildCertificatePairTask(c, cert, pkiDir, cert, &owner) + if err != nil { + return err + } + } + return nil } diff --git a/pkg/apis/nodeup/bootstrap.go b/pkg/apis/nodeup/bootstrap.go index e2b8119b7b761..095718233d77b 100644 --- a/pkg/apis/nodeup/bootstrap.go +++ b/pkg/apis/nodeup/bootstrap.go @@ -22,8 +22,12 @@ const BootstrapAPIVersion = "bootstrap.kops.k8s.io/v1alpha1" type BootstrapRequest struct { // APIVersion defines the versioned schema of this representation of a request. APIVersion string `json:"apiVersion"` + // Certs are the requested certificates and their respective public keys. + Certs map[string]string `json:"certs"` } // BootstrapRespose is a response to a BootstrapRequest. type BootstrapResponse struct { + // Certs are the issued certificates. + Certs map[string]string } diff --git a/pkg/pki/issue.go b/pkg/pki/issue.go index ccf4238973a09..a35e00b3170b5 100644 --- a/pkg/pki/issue.go +++ b/pkg/pki/issue.go @@ -17,6 +17,7 @@ limitations under the License. package pki import ( + "crypto" "crypto/x509" "crypto/x509/pkix" "fmt" @@ -43,7 +44,9 @@ type IssueCertRequest struct { // AlternateNames is a list of alternative names for this certificate. AlternateNames []string - // PrivateKey is the private key for this certificate. If nil, a new private key will be generated. + // PublicKey is the public key for this certificate. If nil, it will be calculated from PrivateKey. + PublicKey crypto.PublicKey + // PrivateKey is the private key for this certificate. If both this and PublicKey are nil, a new private key will be generated. PrivateKey *PrivateKey // Validity is the certificate validity. The default is 10 years. Validity time.Duration @@ -130,7 +133,9 @@ func IssueCert(request *IssueCertRequest, keystore Keystore) (issuedCertificate } privateKey := request.PrivateKey - if privateKey == nil { + if request.PublicKey != nil { + template.PublicKey = request.PublicKey + } else if privateKey == nil { var err error privateKey, err = GeneratePrivateKey() if err != nil { diff --git a/upup/pkg/fi/cloudup/template_functions.go b/upup/pkg/fi/cloudup/template_functions.go index bb598db88b24d..67066f11d13d4 100644 --- a/upup/pkg/fi/cloudup/template_functions.go +++ b/upup/pkg/fi/cloudup/template_functions.go @@ -388,6 +388,8 @@ func (tf *TemplateFunctions) KopsControllerConfig() (string, error) { Listen: fmt.Sprintf(":%d", wellknownports.KopsControllerPort), ServerCertificatePath: path.Join(pkiDir, "kops-controller.crt"), ServerKeyPath: path.Join(pkiDir, "kops-controller.key"), + CABasePath: pkiDir, + SigningCAs: []string{fi.CertificateIDCA}, } switch kops.CloudProviderID(cluster.Spec.CloudProvider) { From c5871df319a8c8c2ebb2d645128d004a50679b38 Mon Sep 17 00:00:00 2001 From: John Gardiner Myers Date: Sat, 1 Aug 2020 22:34:43 -0700 Subject: [PATCH 08/13] Get kubelet certificate from kops-controller --- nodeup/pkg/model/bootstrap_client.go | 7 ++ nodeup/pkg/model/context.go | 47 +++++++++++++ nodeup/pkg/model/kubelet.go | 37 ++++------ pkg/model/components/apiserver.go | 3 +- pkg/model/context.go | 2 +- pkg/model/iam/BUILD.bazel | 1 + pkg/model/iam/iam_builder.go | 9 ++- pkg/model/pki.go | 4 +- upup/pkg/fi/nodeup/command.go | 2 +- .../fi/nodeup/nodetasks/bootstrap_client.go | 69 ++++++++++++++++--- 10 files changed, 139 insertions(+), 42 deletions(-) diff --git a/nodeup/pkg/model/bootstrap_client.go b/nodeup/pkg/model/bootstrap_client.go index 77bd8dce68977..9764e50e928ec 100644 --- a/nodeup/pkg/model/bootstrap_client.go +++ b/nodeup/pkg/model/bootstrap_client.go @@ -55,7 +55,14 @@ func (b BootstrapClientBuilder) Build(c *fi.ModelBuilderContext) error { bootstrapClient := &nodetasks.BootstrapClient{ Authenticator: authenticator, CA: cert, + Certs: b.bootstrapCerts, } + + for _, cert := range b.bootstrapCerts { + cert.Cert.Task = bootstrapClient + cert.Key.Task = bootstrapClient + } + c.AddTask(bootstrapClient) return nil } diff --git a/nodeup/pkg/model/context.go b/nodeup/pkg/model/context.go index fceb1ec8a1a5e..5733cfaba185f 100644 --- a/nodeup/pkg/model/context.go +++ b/nodeup/pkg/model/context.go @@ -57,6 +57,7 @@ type NodeupModelContext struct { IsMaster bool kubernetesVersion semver.Version + bootstrapCerts map[string]*nodetasks.BootstrapCert } // Init completes initialization of the object, for example pre-parsing the kubernetes version @@ -66,6 +67,7 @@ func (c *NodeupModelContext) Init() error { return fmt.Errorf("unable to parse KubernetesVersion %q", c.Cluster.Spec.KubernetesVersion) } c.kubernetesVersion = *k8sVersion + c.bootstrapCerts = map[string]*nodetasks.BootstrapCert{} if c.NodeupConfig.InstanceGroupRole == kops.InstanceGroupRoleMaster { c.IsMaster = true @@ -218,6 +220,51 @@ func (c *NodeupModelContext) BuildIssuedKubeconfig(name string, subject nodetask return kubeConfig.GetConfig() } +// BuildBootstrapKubeconfig generates a kubeconfig with a client certificate from either kops-controller or the state store. +func (c *NodeupModelContext) BuildBootstrapKubeconfig(name string, ctx *fi.ModelBuilderContext) (fi.Resource, error) { + if c.UseKopsControllerForNodeBootstrap() { + b, ok := c.bootstrapCerts[name] + if !ok { + b = &nodetasks.BootstrapCert{ + Cert: &fi.TaskDependentResource{}, + Key: &fi.TaskDependentResource{}, + } + c.bootstrapCerts[name] = b + } + + ca, err := c.GetCert(fi.CertificateIDCA) + if err != nil { + return nil, err + } + + kubeConfig := &nodetasks.KubeConfig{ + Name: name, + Cert: b.Cert, + Key: b.Key, + CA: fi.NewBytesResource(ca), + } + if c.IsMaster { + // @note: use https even for local connections, so we can turn off the insecure port + kubeConfig.ServerURL = "https://127.0.0.1" + } else { + kubeConfig.ServerURL = "https://" + c.Cluster.Spec.MasterInternalName + } + + err = ctx.EnsureTask(kubeConfig) + if err != nil { + return nil, err + } + + return kubeConfig.GetConfig(), nil + } else { + config, err := c.BuildPKIKubeconfig(name) + if err != nil { + return nil, err + } + return fi.NewStringResource(config), nil + } +} + // BuildPKIKubeconfig generates a kubeconfig func (c *NodeupModelContext) BuildPKIKubeconfig(name string) (string, error) { ca, err := c.GetCert(fi.CertificateIDCA) diff --git a/nodeup/pkg/model/kubelet.go b/nodeup/pkg/model/kubelet.go index bf6d01788ae57..e4543547abda4 100644 --- a/nodeup/pkg/model/kubelet.go +++ b/nodeup/pkg/model/kubelet.go @@ -92,7 +92,10 @@ func (b *KubeletBuilder) Build(c *fi.ModelBuilderContext) error { if err != nil { return err } - c.EnsureTask(t) + err = c.EnsureTask(t) + if err != nil { + return err + } } } { @@ -103,26 +106,20 @@ func (b *KubeletBuilder) Build(c *fi.ModelBuilderContext) error { Mode: s("0755"), }) - // @check if bootstrap tokens are enabled and create the appropreiate certificates - if b.UseBootstrapTokens() { - // @check if a master and if so, we bypass the token strapping and instead generate our own kubeconfig + if b.IsMaster || !b.UseBootstrapTokens() { + var kubeconfig fi.Resource if b.IsMaster { - klog.V(3).Info("kubelet bootstrap tokens are enabled and running on a master") - - err := b.buildMasterKubeletKubeconfig(c) - if err != nil { - return err - } + kubeconfig, err = b.buildMasterKubeletKubeconfig(c) + } else { + kubeconfig, err = b.BuildBootstrapKubeconfig("kubelet", c) } - } else { - kubeconfig, err := b.BuildPKIKubeconfig("kubelet") if err != nil { return err } c.AddTask(&nodetasks.File{ Path: b.KubeletKubeConfig(), - Contents: fi.NewStringResource(kubeconfig), + Contents: kubeconfig, Type: nodetasks.FileType_File, Mode: s("0400"), }) @@ -525,23 +522,15 @@ func (b *KubeletBuilder) buildKubeletConfigSpec() (*kops.KubeletConfigSpec, erro } // buildMasterKubeletKubeconfig builds a kubeconfig for the master kubelet, self-signing the kubelet cert -func (b *KubeletBuilder) buildMasterKubeletKubeconfig(c *fi.ModelBuilderContext) error { +func (b *KubeletBuilder) buildMasterKubeletKubeconfig(c *fi.ModelBuilderContext) (fi.Resource, error) { nodeName, err := b.NodeName() if err != nil { - return fmt.Errorf("error getting NodeName: %v", err) + return nil, fmt.Errorf("error getting NodeName: %v", err) } certName := nodetasks.PKIXName{ CommonName: fmt.Sprintf("system:node:%s", nodeName), Organization: []string{rbac.NodesGroup}, } - kubeconfig := b.BuildIssuedKubeconfig("kubelet", certName, c) - c.AddTask(&nodetasks.File{ - Path: b.KubeletKubeConfig(), - Contents: kubeconfig, - Type: nodetasks.FileType_File, - Mode: s("600"), - }) - - return nil + return b.BuildIssuedKubeconfig("kubelet", certName, c), nil } diff --git a/pkg/model/components/apiserver.go b/pkg/model/components/apiserver.go index 3e960d10fe256..f290d25abc043 100644 --- a/pkg/model/components/apiserver.go +++ b/pkg/model/components/apiserver.go @@ -85,8 +85,9 @@ func (b *KubeAPIServerOptionsBuilder) BuildOptions(o interface{}) error { } else if clusterSpec.Authorization.RBAC != nil { var modes []string - if fi.BoolValue(clusterSpec.KubeAPIServer.EnableBootstrapAuthToken) { + if b.IsKubernetesGTE("1.19") || fi.BoolValue(clusterSpec.KubeAPIServer.EnableBootstrapAuthToken) { // Enable the Node authorizer, used for special per-node RBAC policies + // Enable by default from 1.19 - it's an important part of limiting blast radius modes = append(modes, "Node") } modes = append(modes, "RBAC") diff --git a/pkg/model/context.go b/pkg/model/context.go index 25ab1ee78f6cd..e51f28de7d707 100644 --- a/pkg/model/context.go +++ b/pkg/model/context.go @@ -274,7 +274,7 @@ func (m *KopsModelContext) UseKopsControllerForNodeBootstrap() bool { // UseBootstrapTokens checks if bootstrap tokens are enabled func (m *KopsModelContext) UseBootstrapTokens() bool { - if m.Cluster.Spec.KubeAPIServer == nil { + if m.Cluster.Spec.KubeAPIServer == nil || m.UseKopsControllerForNodeBootstrap() { return false } diff --git a/pkg/model/iam/BUILD.bazel b/pkg/model/iam/BUILD.bazel index 96e657ca95aa0..dd2060d457e4f 100644 --- a/pkg/model/iam/BUILD.bazel +++ b/pkg/model/iam/BUILD.bazel @@ -10,6 +10,7 @@ go_library( visibility = ["//visibility:public"], deps = [ "//pkg/apis/kops:go_default_library", + "//pkg/apis/kops/model:go_default_library", "//pkg/util/stringorslice:go_default_library", "//upup/pkg/fi:go_default_library", "//upup/pkg/fi/cloudup/awstasks:go_default_library", diff --git a/pkg/model/iam/iam_builder.go b/pkg/model/iam/iam_builder.go index 6ed0c1b9f4ad3..aa58a252d89ef 100644 --- a/pkg/model/iam/iam_builder.go +++ b/pkg/model/iam/iam_builder.go @@ -35,6 +35,7 @@ import ( "k8s.io/apimachinery/pkg/util/sets" "k8s.io/klog" + "k8s.io/kops/pkg/apis/kops/model" "k8s.io/kops/pkg/apis/kops" "k8s.io/kops/pkg/util/stringorslice" @@ -458,7 +459,9 @@ func ReadableStatePaths(cluster *kops.Cluster, role kops.InstanceGroupRole) ([]s ) // @check if bootstrap tokens are enabled and if so enable access to client certificate - if UseBootstrapTokens(cluster) { + if model.UseKopsControllerForNodeBootstrap(cluster) { + // no additional permissions + } else if useBootstrapTokens(cluster) { paths = append(paths, "/pki/private/node-authorizer-client/*") } else { paths = append(paths, "/pki/private/kubelet/*") @@ -539,9 +542,9 @@ func (b *PolicyResource) Open() (io.Reader, error) { return bytes.NewReader([]byte(j)), nil } -// UseBootstrapTokens check if we are using bootstrap tokens - @TODO, i don't like this we should probably pass in +// useBootstrapTokens check if we are using bootstrap tokens - @TODO, i don't like this we should probably pass in // the kops model into the builder rather than duplicating the code. I'll leave for another PR -func UseBootstrapTokens(cluster *kops.Cluster) bool { +func useBootstrapTokens(cluster *kops.Cluster) bool { if cluster.Spec.KubeAPIServer == nil { return false } diff --git a/pkg/model/pki.go b/pkg/model/pki.go index e3a954e09a006..db9a09e6c6277 100644 --- a/pkg/model/pki.go +++ b/pkg/model/pki.go @@ -48,9 +48,9 @@ func (b *PKIModelBuilder) Build(c *fi.ModelBuilderContext) error { c.AddTask(defaultCA) { - // @check of bootstrap tokens are enable if so, disable the creation of the kubelet certificate - we also + // @check if kops-controller bootstrap or bootstrap tokens are enabled. If so, disable the creation of the kubelet certificate - we also // block at the IAM level for AWS cluster for pre-existing clusters. - if !b.UseBootstrapTokens() { + if !b.UseKopsControllerForNodeBootstrap() && !b.UseBootstrapTokens() { c.AddTask(&fitasks.Keypair{ Name: fi.String("kubelet"), Lifecycle: b.Lifecycle, diff --git a/upup/pkg/fi/nodeup/command.go b/upup/pkg/fi/nodeup/command.go index f75666da7d874..9db3c700bd9a2 100644 --- a/upup/pkg/fi/nodeup/command.go +++ b/upup/pkg/fi/nodeup/command.go @@ -218,7 +218,6 @@ func (c *NodeUpCommand) Run(out io.Writer) error { } loader := &Loader{} - loader.Builders = append(loader.Builders, &model.BootstrapClientBuilder{NodeupModelContext: modelContext}) loader.Builders = append(loader.Builders, &model.NTPBuilder{NodeupModelContext: modelContext}) loader.Builders = append(loader.Builders, &model.MiscUtilsBuilder{NodeupModelContext: modelContext}) loader.Builders = append(loader.Builders, &model.DirectoryBuilder{NodeupModelContext: modelContext}) @@ -253,6 +252,7 @@ func (c *NodeUpCommand) Run(out io.Writer) error { loader.Builders = append(loader.Builders, &networking.KuberouterBuilder{NodeupModelContext: modelContext}) loader.Builders = append(loader.Builders, &networking.LyftVPCBuilder{NodeupModelContext: modelContext}) + loader.Builders = append(loader.Builders, &model.BootstrapClientBuilder{NodeupModelContext: modelContext}) taskMap, err := loader.Build() if err != nil { return fmt.Errorf("error building loader: %v", err) diff --git a/upup/pkg/fi/nodeup/nodetasks/bootstrap_client.go b/upup/pkg/fi/nodeup/nodetasks/bootstrap_client.go index 5844b572360f8..6bf19d00337b6 100644 --- a/upup/pkg/fi/nodeup/nodetasks/bootstrap_client.go +++ b/upup/pkg/fi/nodeup/nodetasks/bootstrap_client.go @@ -19,9 +19,11 @@ package nodetasks import ( "bufio" "bytes" + "crypto/rsa" "crypto/tls" "crypto/x509" "encoding/json" + "encoding/pem" "fmt" "io/ioutil" "net" @@ -30,6 +32,7 @@ import ( "strconv" "k8s.io/kops/pkg/apis/nodeup" + "k8s.io/kops/pkg/pki" "k8s.io/kops/pkg/wellknownports" "k8s.io/kops/upup/pkg/fi" ) @@ -39,8 +42,16 @@ type BootstrapClient struct { Authenticator fi.Authenticator // CA is the CA certificate for kops-controller. CA []byte + // Certs are the requested certificates. + Certs map[string]*BootstrapCert client *http.Client + keys map[string]*pki.PrivateKey +} + +type BootstrapCert struct { + Cert *fi.TaskDependentResource + Key *fi.TaskDependentResource } var _ fi.Task = &BootstrapClient{} @@ -63,17 +74,55 @@ func (b *BootstrapClient) String() string { func (b *BootstrapClient) Run(c *fi.Context) error { req := nodeup.BootstrapRequest{ APIVersion: nodeup.BootstrapAPIVersion, + Certs: map[string]string{}, + } + + if b.keys == nil { + b.keys = map[string]*pki.PrivateKey{} + } + + for name, certRequest := range b.Certs { + key, ok := b.keys[name] + if !ok { + var err error + key, err = pki.GeneratePrivateKey() + if err != nil { + return fmt.Errorf("generating private key: %v", err) + } + + certRequest.Key.Resource = &asBytesResource{key} + b.keys[name] = key + } + + pkData, err := x509.MarshalPKIXPublicKey(key.Key.(*rsa.PrivateKey).Public()) + if err != nil { + return fmt.Errorf("marshalling public key: %v", err) + } + // TODO perhaps send a CSR instead to prove we own the private key? + req.Certs[name] = string(pem.EncodeToMemory(&pem.Block{Type: "RSA PUBLIC KEY", Bytes: pkData})) } - err := b.queryBootstrap(c, req) + resp, err := b.queryBootstrap(c, &req) if err != nil { return err } + for name, certRequest := range b.Certs { + cert, ok := resp.Certs[name] + if !ok { + return fmt.Errorf("kops-controller did not return a %q certificate", name) + } + certificate, err := pki.ParsePEMCertificate([]byte(cert)) + if err != nil { + return fmt.Errorf("parsing %q certificate: %v", name, err) + } + certRequest.Cert.Resource = asBytesResource{certificate} + } + return nil } -func (b *BootstrapClient) queryBootstrap(c *fi.Context, req nodeup.BootstrapRequest) error { +func (b *BootstrapClient) queryBootstrap(c *fi.Context, req *nodeup.BootstrapRequest) (*nodeup.BootstrapResponse, error) { if b.client == nil { certPool := x509.NewCertPool() certPool.AppendCertsFromPEM(b.CA) @@ -90,7 +139,7 @@ func (b *BootstrapClient) queryBootstrap(c *fi.Context, req nodeup.BootstrapRequ reqBytes, err := json.Marshal(req) if err != nil { - return err + return nil, err } bootstrapUrl := url.URL{ @@ -100,19 +149,19 @@ func (b *BootstrapClient) queryBootstrap(c *fi.Context, req nodeup.BootstrapRequ } httpReq, err := http.NewRequest("POST", bootstrapUrl.String(), bytes.NewReader(reqBytes)) if err != nil { - return err + return nil, err } httpReq.Header.Set("Content-Type", "application/json") token, err := b.Authenticator.CreateToken(reqBytes) if err != nil { - return err + return nil, err } httpReq.Header.Set("Authorization", token) resp, err := b.client.Do(httpReq) if err != nil { - return err + return nil, err } if resp.Body != nil { defer resp.Body.Close() @@ -126,19 +175,19 @@ func (b *BootstrapClient) queryBootstrap(c *fi.Context, req nodeup.BootstrapRequ detail = scanner.Text() } } - return fmt.Errorf("bootstrap returned status code %d: %s", resp.StatusCode, detail) + return nil, fmt.Errorf("bootstrap returned status code %d: %s", resp.StatusCode, detail) } var bootstrapResp nodeup.BootstrapResponse body, err := ioutil.ReadAll(resp.Body) if err != nil { - return err + return nil, err } err = json.Unmarshal(body, &bootstrapResp) if err != nil { - return err + return nil, err } - return nil + return &bootstrapResp, nil } From fb381c4c8be7df135d7c2e66ab25c8e70c018281 Mon Sep 17 00:00:00 2001 From: John Gardiner Myers Date: Mon, 3 Aug 2020 20:49:50 -0700 Subject: [PATCH 09/13] Don't issue kubelet cert on masters before k8s 1.19 --- nodeup/pkg/model/kubelet.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nodeup/pkg/model/kubelet.go b/nodeup/pkg/model/kubelet.go index e4543547abda4..6bbaa44ad1a7d 100644 --- a/nodeup/pkg/model/kubelet.go +++ b/nodeup/pkg/model/kubelet.go @@ -108,7 +108,7 @@ func (b *KubeletBuilder) Build(c *fi.ModelBuilderContext) error { if b.IsMaster || !b.UseBootstrapTokens() { var kubeconfig fi.Resource - if b.IsMaster { + if b.IsMaster && (b.IsKubernetesGTE("1.19") || b.UseBootstrapTokens()) { kubeconfig, err = b.buildMasterKubeletKubeconfig(c) } else { kubeconfig, err = b.BuildBootstrapKubeconfig("kubelet", c) From 8ec55f28c7b41971fa4f26302612cc40bd9dd12f Mon Sep 17 00:00:00 2001 From: John Gardiner Myers Date: Mon, 3 Aug 2020 22:19:28 -0700 Subject: [PATCH 10/13] Don't install RBAC addon when using kops-controller bootstrap --- upup/pkg/fi/cloudup/bootstrapchannelbuilder.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/upup/pkg/fi/cloudup/bootstrapchannelbuilder.go b/upup/pkg/fi/cloudup/bootstrapchannelbuilder.go index 81356e5763e86..2f3f27df250f6 100644 --- a/upup/pkg/fi/cloudup/bootstrapchannelbuilder.go +++ b/upup/pkg/fi/cloudup/bootstrapchannelbuilder.go @@ -304,7 +304,7 @@ func (b *BootstrapChannelBuilder) buildAddons() *channelsapi.Addons { // this manifest. For clusters whom are upgrading from RBAC to Node,RBAC the clusterrolebinding // will remain and have to be deleted manually once all the nodes have been upgraded. enableRBACAddon := true - if b.Cluster.Spec.NodeAuthorization != nil { + if b.UseKopsControllerForNodeBootstrap() || b.Cluster.Spec.NodeAuthorization != nil { enableRBACAddon = false } if b.Cluster.Spec.KubeAPIServer != nil { From 9e99f76a6ea33a7b2ff3ee8d9c06a1d9aa8890ef Mon Sep 17 00:00:00 2001 From: John Gardiner Myers Date: Sat, 8 Aug 2020 16:10:05 -0700 Subject: [PATCH 11/13] Address review comments --- cmd/kops-controller/pkg/server/BUILD.bazel | 1 - cmd/kops-controller/pkg/server/server.go | 7 +++---- upup/pkg/fi/authenticate.go | 4 ++-- upup/pkg/fi/cloudup/awsup/aws_authenticator.go | 2 ++ upup/pkg/fi/cloudup/awsup/aws_verifier.go | 4 ++-- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/cmd/kops-controller/pkg/server/BUILD.bazel b/cmd/kops-controller/pkg/server/BUILD.bazel index 522bbaba93d77..026e18d27fc88 100644 --- a/cmd/kops-controller/pkg/server/BUILD.bazel +++ b/cmd/kops-controller/pkg/server/BUILD.bazel @@ -14,7 +14,6 @@ go_library( "//pkg/pki:go_default_library", "//pkg/rbac:go_default_library", "//upup/pkg/fi:go_default_library", - "//vendor/github.com/gorilla/mux:go_default_library", "//vendor/k8s.io/klog:go_default_library", ], ) diff --git a/cmd/kops-controller/pkg/server/server.go b/cmd/kops-controller/pkg/server/server.go index 7a28de8ac3baa..72fea09c1d6e0 100644 --- a/cmd/kops-controller/pkg/server/server.go +++ b/cmd/kops-controller/pkg/server/server.go @@ -29,7 +29,6 @@ import ( "runtime/debug" "time" - "github.com/gorilla/mux" "k8s.io/klog" "k8s.io/kops/cmd/kops-controller/pkg/config" "k8s.io/kops/pkg/apis/nodeup" @@ -59,7 +58,7 @@ func NewServer(opt *config.Options, verifier fi.Verifier) (*Server, error) { server: server, verifier: verifier, } - r := mux.NewRouter() + r := http.NewServeMux() r.Handle("/bootstrap", http.HandlerFunc(s.bootstrap)) server.Handler = recovery(r) @@ -139,7 +138,7 @@ func (s *Server) bootstrap(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") _ = json.NewEncoder(w).Encode(resp) - klog.Infof("bootstrap %s %s success", r.RemoteAddr, id.Instance) + klog.Infof("bootstrap %s %s success", r.RemoteAddr, id.NodeName) } func (s *Server) issueCert(name string, pubKey string, id *fi.VerifyResult, validHours uint32) (string, error) { @@ -162,7 +161,7 @@ func (s *Server) issueCert(name string, pubKey string, id *fi.VerifyResult, vali switch name { case "kubelet": issueReq.Subject = pkix.Name{ - CommonName: fmt.Sprintf("system:node:%s", id.Instance), + CommonName: fmt.Sprintf("system:node:%s", id.NodeName), Organization: []string{rbac.NodesGroup}, } default: diff --git a/upup/pkg/fi/authenticate.go b/upup/pkg/fi/authenticate.go index 2e96bac4e252b..3edd2a1f271b6 100644 --- a/upup/pkg/fi/authenticate.go +++ b/upup/pkg/fi/authenticate.go @@ -23,8 +23,8 @@ type Authenticator interface { // VerifyResult is the result of a successfully verified request. type VerifyResult struct { - // Instance is the hostname of the instance. - Instance string + // Nodename is the name that this node is authorized to use. + NodeName string } // Verifier verifies authentication credentials for requests. diff --git a/upup/pkg/fi/cloudup/awsup/aws_authenticator.go b/upup/pkg/fi/cloudup/awsup/aws_authenticator.go index 7f3e1bc7cf5d5..ff304a5005911 100644 --- a/upup/pkg/fi/cloudup/awsup/aws_authenticator.go +++ b/upup/pkg/fi/cloudup/awsup/aws_authenticator.go @@ -51,7 +51,9 @@ func (a awsAuthenticator) CreateToken(body []byte) (string, error) { stsRequest, _ := a.sts.GetCallerIdentityRequest(nil) + // Ensure the signature is only valid for this particular body content. stsRequest.HTTPRequest.Header.Add("X-Kops-Request-SHA", base64.RawStdEncoding.EncodeToString(sha[:])) + err := stsRequest.Sign() if err != nil { return "", err diff --git a/upup/pkg/fi/cloudup/awsup/aws_verifier.go b/upup/pkg/fi/cloudup/awsup/aws_verifier.go index b19852a40fa9c..08d82a3b77844 100644 --- a/upup/pkg/fi/cloudup/awsup/aws_verifier.go +++ b/upup/pkg/fi/cloudup/awsup/aws_verifier.go @@ -139,6 +139,7 @@ func (a awsVerifier) VerifyToken(token string, body []byte) (*fi.VerifyResult, e return nil, fmt.Errorf("unmarshalling authorization token: %v", err) } + // Verify the token has signed the body content. sha := sha256.Sum256(body) if stsRequest.HTTPRequest.Header.Get("X-Kops-Request-SHA") != base64.RawStdEncoding.EncodeToString(sha[:]) { return nil, fmt.Errorf("incorrect SHA") @@ -150,7 +151,6 @@ func (a awsVerifier) VerifyToken(token string, body []byte) (*fi.VerifyResult, e return nil, fmt.Errorf("incorrect content-length") } - // TODO - implement retry? response, err := a.client.Do(stsRequest.HTTPRequest) if err != nil { return nil, fmt.Errorf("sending STS request: %v", err) @@ -225,6 +225,6 @@ func (a awsVerifier) VerifyToken(token string, body []byte) (*fi.VerifyResult, e } return &fi.VerifyResult{ - Instance: aws.StringValue(instances.Reservations[0].Instances[0].PrivateDnsName), + NodeName: aws.StringValue(instances.Reservations[0].Instances[0].PrivateDnsName), }, nil } From 5dcc95178ffb0be93b98c2275342dfea8e17571c Mon Sep 17 00:00:00 2001 From: John Gardiner Myers Date: Sun, 9 Aug 2020 11:39:18 -0700 Subject: [PATCH 12/13] Verify the ARN's partition --- upup/pkg/fi/cloudup/awsup/aws_verifier.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/upup/pkg/fi/cloudup/awsup/aws_verifier.go b/upup/pkg/fi/cloudup/awsup/aws_verifier.go index 08d82a3b77844..e9bd9552dc64d 100644 --- a/upup/pkg/fi/cloudup/awsup/aws_verifier.go +++ b/upup/pkg/fi/cloudup/awsup/aws_verifier.go @@ -46,6 +46,7 @@ type AWSVerifierOptions struct { type awsVerifier struct { accountId string + partition string opt AWSVerifierOptions ec2 *ec2.EC2 @@ -68,6 +69,8 @@ func NewAWSVerifier(opt *AWSVerifierOptions) (fi.Verifier, error) { return nil, err } + partition := strings.Split(aws.StringValue(identity.Arn), ":")[1] + metadata := ec2metadata.New(sess, config) region, err := metadata.Region() if err != nil { @@ -78,6 +81,7 @@ func NewAWSVerifier(opt *AWSVerifierOptions) (fi.Verifier, error) { return &awsVerifier{ accountId: aws.StringValue(identity.Account), + partition: partition, opt: *opt, ec2: ec2Client, sts: stsClient, @@ -185,7 +189,9 @@ func (a awsVerifier) VerifyToken(token string, body []byte) (*fi.VerifyResult, e if parts[0] != "arn" { return nil, fmt.Errorf("arn %q doesn't start with \"arn:\"", arn) } - // parts[1] is partition + if parts[1] != a.partition { + return nil, fmt.Errorf("arn %q not in partion %q", arn, a.partition) + } if parts[2] != "iam" && parts[2] != "sts" { return nil, fmt.Errorf("arn %q has unrecognized service", arn) } From 1a253dc574628e4598dc87a3512ebfb05d189cf2 Mon Sep 17 00:00:00 2001 From: John Gardiner Myers Date: Sun, 9 Aug 2020 14:06:55 -0700 Subject: [PATCH 13/13] Send the STS queries to the local region --- nodeup/pkg/model/bootstrap_client.go | 6 +++++- upup/pkg/fi/cloudup/awsup/BUILD.bazel | 1 - upup/pkg/fi/cloudup/awsup/aws_authenticator.go | 4 ++-- upup/pkg/fi/cloudup/awsup/aws_verifier.go | 13 ++++--------- upup/pkg/fi/cloudup/template_functions.go | 1 + 5 files changed, 12 insertions(+), 13 deletions(-) diff --git a/nodeup/pkg/model/bootstrap_client.go b/nodeup/pkg/model/bootstrap_client.go index 9764e50e928ec..f3daf2bd4660d 100644 --- a/nodeup/pkg/model/bootstrap_client.go +++ b/nodeup/pkg/model/bootstrap_client.go @@ -39,7 +39,11 @@ func (b BootstrapClientBuilder) Build(c *fi.ModelBuilderContext) error { var err error switch kops.CloudProviderID(b.Cluster.Spec.CloudProvider) { case kops.CloudProviderAWS: - authenticator, err = awsup.NewAWSAuthenticator() + region, regionErr := awsup.FindRegion(b.Cluster) + if regionErr != nil { + return fmt.Errorf("querying AWS region: %v", regionErr) + } + authenticator, err = awsup.NewAWSAuthenticator(region) default: return fmt.Errorf("unsupported cloud provider %s", b.Cluster.Spec.CloudProvider) } diff --git a/upup/pkg/fi/cloudup/awsup/BUILD.bazel b/upup/pkg/fi/cloudup/awsup/BUILD.bazel index 193241329503a..4d3dca315b77b 100644 --- a/upup/pkg/fi/cloudup/awsup/BUILD.bazel +++ b/upup/pkg/fi/cloudup/awsup/BUILD.bazel @@ -30,7 +30,6 @@ go_library( "//vendor/github.com/aws/aws-sdk-go/aws:go_default_library", "//vendor/github.com/aws/aws-sdk-go/aws/awserr:go_default_library", "//vendor/github.com/aws/aws-sdk-go/aws/client:go_default_library", - "//vendor/github.com/aws/aws-sdk-go/aws/ec2metadata:go_default_library", "//vendor/github.com/aws/aws-sdk-go/aws/endpoints:go_default_library", "//vendor/github.com/aws/aws-sdk-go/aws/request:go_default_library", "//vendor/github.com/aws/aws-sdk-go/aws/session:go_default_library", diff --git a/upup/pkg/fi/cloudup/awsup/aws_authenticator.go b/upup/pkg/fi/cloudup/awsup/aws_authenticator.go index ff304a5005911..9a471331d5765 100644 --- a/upup/pkg/fi/cloudup/awsup/aws_authenticator.go +++ b/upup/pkg/fi/cloudup/awsup/aws_authenticator.go @@ -35,8 +35,8 @@ type awsAuthenticator struct { var _ fi.Authenticator = &awsAuthenticator{} -func NewAWSAuthenticator() (fi.Authenticator, error) { - config := aws.NewConfig().WithCredentialsChainVerboseErrors(true) +func NewAWSAuthenticator(region string) (fi.Authenticator, error) { + config := aws.NewConfig().WithCredentialsChainVerboseErrors(true).WithRegion(region) sess, err := session.NewSession(config) if err != nil { return nil, err diff --git a/upup/pkg/fi/cloudup/awsup/aws_verifier.go b/upup/pkg/fi/cloudup/awsup/aws_verifier.go index e9bd9552dc64d..cf4e14e884294 100644 --- a/upup/pkg/fi/cloudup/awsup/aws_verifier.go +++ b/upup/pkg/fi/cloudup/awsup/aws_verifier.go @@ -32,7 +32,6 @@ import ( "time" "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/ec2metadata" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/ec2" "github.com/aws/aws-sdk-go/service/sts" @@ -42,6 +41,8 @@ import ( type AWSVerifierOptions struct { // NodesRoles are the IAM roles that worker nodes are permitted to have. NodesRoles []string `json:"nodesRoles"` + // Region is the AWS region of the cluster. + Region string } type awsVerifier struct { @@ -57,7 +58,7 @@ type awsVerifier struct { var _ fi.Verifier = &awsVerifier{} func NewAWSVerifier(opt *AWSVerifierOptions) (fi.Verifier, error) { - config := aws.NewConfig().WithCredentialsChainVerboseErrors(true) + config := aws.NewConfig().WithCredentialsChainVerboseErrors(true).WithRegion(opt.Region) sess, err := session.NewSession(config) if err != nil { return nil, err @@ -71,13 +72,7 @@ func NewAWSVerifier(opt *AWSVerifierOptions) (fi.Verifier, error) { partition := strings.Split(aws.StringValue(identity.Arn), ":")[1] - metadata := ec2metadata.New(sess, config) - region, err := metadata.Region() - if err != nil { - return nil, fmt.Errorf("error querying ec2 metadata service (for region): %v", err) - } - - ec2Client := ec2.New(sess, config.WithRegion(region)) + ec2Client := ec2.New(sess, config) return &awsVerifier{ accountId: aws.StringValue(identity.Account), diff --git a/upup/pkg/fi/cloudup/template_functions.go b/upup/pkg/fi/cloudup/template_functions.go index 67066f11d13d4..bd5a6e68cc9c2 100644 --- a/upup/pkg/fi/cloudup/template_functions.go +++ b/upup/pkg/fi/cloudup/template_functions.go @@ -406,6 +406,7 @@ func (tf *TemplateFunctions) KopsControllerConfig() (string, error) { } config.Server.Provider.AWS = &awsup.AWSVerifierOptions{ NodesRoles: nodesRoles.List(), + Region: tf.Region, } default: return "", fmt.Errorf("unsupported cloud provider %s", cluster.Spec.CloudProvider)