diff --git a/cmd/kops-controller/BUILD.bazel b/cmd/kops-controller/BUILD.bazel index 756dd7b64c6f6..6e921b82d0573 100644 --- a/cmd/kops-controller/BUILD.bazel +++ b/cmd/kops-controller/BUILD.bazel @@ -8,11 +8,14 @@ go_library( deps = [ "//cmd/kops-controller/controllers:go_default_library", "//cmd/kops-controller/pkg/config:go_default_library", + "//cmd/kops-controller/pkg/server:go_default_library", "//pkg/nodeidentity:go_default_library", "//pkg/nodeidentity/aws:go_default_library", "//pkg/nodeidentity/do:go_default_library", "//pkg/nodeidentity/gce:go_default_library", "//pkg/nodeidentity/openstack:go_default_library", + "//upup/pkg/fi:go_default_library", + "//upup/pkg/fi/cloudup/awsup:go_default_library", "//vendor/k8s.io/api/core/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", "//vendor/k8s.io/client-go/plugin/pkg/client/auth/gcp:go_default_library", @@ -47,7 +50,7 @@ ARCH = [ architecture = arch, base = "@distroless_base//image", cmd = ["/kops-controller"], - user = "10001", + user = "10011", files = [ "//cmd/kops-controller", ], diff --git a/cmd/kops-controller/main.go b/cmd/kops-controller/main.go index c1a4a7fd2c3f1..0ff547323c0d2 100644 --- a/cmd/kops-controller/main.go +++ b/cmd/kops-controller/main.go @@ -29,11 +29,14 @@ import ( "k8s.io/klog/klogr" "k8s.io/kops/cmd/kops-controller/controllers" "k8s.io/kops/cmd/kops-controller/pkg/config" + "k8s.io/kops/cmd/kops-controller/pkg/server" "k8s.io/kops/pkg/nodeidentity" nodeidentityaws "k8s.io/kops/pkg/nodeidentity/aws" nodeidentitydo "k8s.io/kops/pkg/nodeidentity/do" nodeidentitygce "k8s.io/kops/pkg/nodeidentity/gce" nodeidentityos "k8s.io/kops/pkg/nodeidentity/openstack" + "k8s.io/kops/upup/pkg/fi" + "k8s.io/kops/upup/pkg/fi/cloudup/awsup" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/yaml" @@ -81,6 +84,30 @@ func main() { } ctrl.SetLogger(klogr.New()) + if opt.Server != nil { + var verifier fi.Verifier + var err error + if opt.Server.Provider.AWS != nil { + verifier, err = awsup.NewAWSVerifier(opt.Server.Provider.AWS) + if err != nil { + setupLog.Error(err, "unable to create verifier") + os.Exit(1) + } + } else { + klog.Fatalf("server cloud provider config not provided") + } + + srv, err := server.NewServer(&opt, verifier) + if err != nil { + setupLog.Error(err, "unable to create server") + os.Exit(1) + } + go func() { + err := srv.Start() + setupLog.Error(err, "unable to start server") + os.Exit(1) + }() + } if err := buildScheme(); err != nil { setupLog.Error(err, "error building scheme") diff --git a/cmd/kops-controller/pkg/config/BUILD.bazel b/cmd/kops-controller/pkg/config/BUILD.bazel index 093d96826049e..ee82bdbfb7d19 100644 --- a/cmd/kops-controller/pkg/config/BUILD.bazel +++ b/cmd/kops-controller/pkg/config/BUILD.bazel @@ -5,4 +5,5 @@ go_library( srcs = ["options.go"], importpath = "k8s.io/kops/cmd/kops-controller/pkg/config", visibility = ["//visibility:public"], + deps = ["//upup/pkg/fi/cloudup/awsup:go_default_library"], ) diff --git a/cmd/kops-controller/pkg/config/options.go b/cmd/kops-controller/pkg/config/options.go index 22a15914b17ca..5a82a3bfbe3e1 100644 --- a/cmd/kops-controller/pkg/config/options.go +++ b/cmd/kops-controller/pkg/config/options.go @@ -16,10 +16,35 @@ limitations under the License. package config +import "k8s.io/kops/upup/pkg/fi/cloudup/awsup" + type Options struct { - Cloud string `json:"cloud,omitempty"` - ConfigBase string `json:"configBase,omitempty"` + Cloud string `json:"cloud,omitempty"` + ConfigBase string `json:"configBase,omitempty"` + Server *ServerOptions `json:"server,omitempty"` } func (o *Options) PopulateDefaults() { } + +type ServerOptions struct { + // Listen is the network endpoint (ip and port) we should listen on. + Listen string + + // Provider is the cloud provider. + Provider ServerProviderOptions `json:"provider"` + + // ServerKeyPath is the path to our TLS serving private key. + ServerKeyPath string `json:"serverKeyPath,omitempty"` + // ServerCertificatePath is the path to our TLS serving certificate. + ServerCertificatePath string `json:"serverCertificatePath,omitempty"` + + // CABasePath is a base of the path to CA certificate and key files. + CABasePath string `json:"caBasePath"` + // SigningCAs is the list of active signing CAs. + SigningCAs []string `json:"signingCAs"` +} + +type ServerProviderOptions struct { + AWS *awsup.AWSVerifierOptions `json:"aws,omitempty"` +} diff --git a/cmd/kops-controller/pkg/server/BUILD.bazel b/cmd/kops-controller/pkg/server/BUILD.bazel new file mode 100644 index 0000000000000..026e18d27fc88 --- /dev/null +++ b/cmd/kops-controller/pkg/server/BUILD.bazel @@ -0,0 +1,19 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "go_default_library", + srcs = [ + "keystore.go", + "server.go", + ], + importpath = "k8s.io/kops/cmd/kops-controller/pkg/server", + visibility = ["//visibility:public"], + deps = [ + "//cmd/kops-controller/pkg/config:go_default_library", + "//pkg/apis/nodeup:go_default_library", + "//pkg/pki:go_default_library", + "//pkg/rbac:go_default_library", + "//upup/pkg/fi:go_default_library", + "//vendor/k8s.io/klog:go_default_library", + ], +) diff --git a/cmd/kops-controller/pkg/server/keystore.go b/cmd/kops-controller/pkg/server/keystore.go new file mode 100644 index 0000000000000..52dfb8a52bea8 --- /dev/null +++ b/cmd/kops-controller/pkg/server/keystore.go @@ -0,0 +1,76 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package server + +import ( + "fmt" + "io/ioutil" + "path" + + "k8s.io/kops/pkg/pki" +) + +type keystore struct { + keys map[string]keystoreEntry +} + +type keystoreEntry struct { + certificate *pki.Certificate + key *pki.PrivateKey +} + +var _ pki.Keystore = keystore{} + +func (k keystore) FindKeypair(name string) (*pki.Certificate, *pki.PrivateKey, bool, error) { + entry, ok := k.keys[name] + if !ok { + return nil, nil, false, fmt.Errorf("unknown CA %q", name) + } + return entry.certificate, entry.key, false, nil +} + +func newKeystore(basePath string, cas []string) (pki.Keystore, error) { + keystore := &keystore{ + keys: map[string]keystoreEntry{}, + } + for _, name := range cas { + certBytes, err := ioutil.ReadFile(path.Join(basePath, name+".pem")) + if err != nil { + return nil, fmt.Errorf("reading %q certificate: %v", name, err) + } + certificate, err := pki.ParsePEMCertificate(certBytes) + if err != nil { + return nil, fmt.Errorf("parsing %q certificate: %v", name, err) + } + + keyBytes, err := ioutil.ReadFile(path.Join(basePath, name+"-key.pem")) + if err != nil { + return nil, fmt.Errorf("reading %q key: %v", name, err) + } + key, err := pki.ParsePEMPrivateKey(keyBytes) + if err != nil { + return nil, fmt.Errorf("parsing %q key: %v", name, err) + } + + keystore.keys[name] = keystoreEntry{ + certificate: certificate, + key: key, + } + } + + return keystore, nil +} diff --git a/cmd/kops-controller/pkg/server/server.go b/cmd/kops-controller/pkg/server/server.go new file mode 100644 index 0000000000000..72fea09c1d6e0 --- /dev/null +++ b/cmd/kops-controller/pkg/server/server.go @@ -0,0 +1,192 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package server + +import ( + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/json" + "encoding/pem" + "fmt" + "hash/fnv" + "io/ioutil" + "net/http" + "runtime/debug" + "time" + + "k8s.io/klog" + "k8s.io/kops/cmd/kops-controller/pkg/config" + "k8s.io/kops/pkg/apis/nodeup" + "k8s.io/kops/pkg/pki" + "k8s.io/kops/pkg/rbac" + "k8s.io/kops/upup/pkg/fi" +) + +type Server struct { + opt *config.Options + server *http.Server + verifier fi.Verifier + keystore pki.Keystore +} + +func NewServer(opt *config.Options, verifier fi.Verifier) (*Server, error) { + server := &http.Server{ + Addr: opt.Server.Listen, + TLSConfig: &tls.Config{ + MinVersion: tls.VersionTLS12, + PreferServerCipherSuites: true, + }, + } + + s := &Server{ + opt: opt, + server: server, + verifier: verifier, + } + r := http.NewServeMux() + r.Handle("/bootstrap", http.HandlerFunc(s.bootstrap)) + server.Handler = recovery(r) + + return s, nil +} + +func (s *Server) Start() error { + var err error + s.keystore, err = newKeystore(s.opt.Server.CABasePath, s.opt.Server.SigningCAs) + if err != nil { + return err + } + + return s.server.ListenAndServeTLS(s.opt.Server.ServerCertificatePath, s.opt.Server.ServerKeyPath) +} + +func (s *Server) bootstrap(w http.ResponseWriter, r *http.Request) { + if r.Body == nil { + klog.Infof("bootstrap %s no body", r.RemoteAddr) + w.WriteHeader(http.StatusBadRequest) + return + } + + body, err := ioutil.ReadAll(r.Body) + if err != nil { + klog.Infof("bootstrap %s read err: %v", r.RemoteAddr, err) + w.WriteHeader(http.StatusBadRequest) + _, _ = w.Write([]byte(fmt.Sprintf("bootstrap %s failed to read body: %v", r.RemoteAddr, err))) + return + } + + id, err := s.verifier.VerifyToken(r.Header.Get("Authorization"), body) + if err != nil { + klog.Infof("bootstrap %s verify err: %v", r.RemoteAddr, err) + w.WriteHeader(http.StatusForbidden) + _, _ = w.Write([]byte(fmt.Sprintf("failed to verify token: %v", err))) + return + } + + req := &nodeup.BootstrapRequest{} + err = json.Unmarshal(body, req) + if err != nil { + klog.Infof("bootstrap %s decode err: %v", r.RemoteAddr, err) + w.WriteHeader(http.StatusBadRequest) + _, _ = w.Write([]byte(fmt.Sprintf("failed to decode: %v", err))) + return + } + + if req.APIVersion != nodeup.BootstrapAPIVersion { + klog.Infof("bootstrap %s wrong APIVersion", r.RemoteAddr) + w.WriteHeader(http.StatusBadRequest) + _, _ = w.Write([]byte("unexpected APIVersion")) + return + } + + resp := &nodeup.BootstrapResponse{ + Certs: map[string]string{}, + } + + // Skew the certificate lifetime by up to 30 days based on information about the requesting node. + // This is so that different nodes created at the same time have the certificates they generated + // expire at different times, but all certificates on a given node expire around the same time. + hash := fnv.New32() + _, _ = hash.Write([]byte(r.RemoteAddr)) + validHours := (455 * 24) + (hash.Sum32() % (30 * 24)) + + for name, pubKey := range req.Certs { + cert, err := s.issueCert(name, pubKey, id, validHours) + if err != nil { + klog.Infof("bootstrap %s cert %q issue err: %v", r.RemoteAddr, name, err) + w.WriteHeader(http.StatusBadRequest) + _, _ = w.Write([]byte(fmt.Sprintf("failed to issue %q: %v", name, err))) + return + } + resp.Certs[name] = cert + } + + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(resp) + klog.Infof("bootstrap %s %s success", r.RemoteAddr, id.NodeName) +} + +func (s *Server) issueCert(name string, pubKey string, id *fi.VerifyResult, validHours uint32) (string, error) { + block, _ := pem.Decode([]byte(pubKey)) + if block.Type != "RSA PUBLIC KEY" { + return "", fmt.Errorf("unexpected key type %q", block.Type) + } + key, err := x509.ParsePKIXPublicKey(block.Bytes) + if err != nil { + return "", fmt.Errorf("parsing key: %v", err) + } + + issueReq := &pki.IssueCertRequest{ + Signer: fi.CertificateIDCA, + Type: "client", + PublicKey: key, + Validity: time.Hour * time.Duration(validHours), + } + + switch name { + case "kubelet": + issueReq.Subject = pkix.Name{ + CommonName: fmt.Sprintf("system:node:%s", id.NodeName), + Organization: []string{rbac.NodesGroup}, + } + default: + return "", fmt.Errorf("unexpected key name") + } + + cert, _, _, err := pki.IssueCert(issueReq, s.keystore) + if err != nil { + return "", fmt.Errorf("issuing certificate: %v", err) + } + + return cert.AsString() +} + +// recovery is responsible for ensuring we don't exit on a panic. +func recovery(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + defer func() { + if err := recover(); err != nil { + w.WriteHeader(http.StatusInternalServerError) + + klog.Errorf("failed to handle request: threw exception: %v: %s", err, debug.Stack()) + } + }() + + next.ServeHTTP(w, req) + }) +} diff --git a/hack/.packages b/hack/.packages index 0c713ab385b96..baaa194f367a7 100644 --- a/hack/.packages +++ b/hack/.packages @@ -20,6 +20,7 @@ k8s.io/kops/cmd/kops/util k8s.io/kops/cmd/kops-controller k8s.io/kops/cmd/kops-controller/controllers k8s.io/kops/cmd/kops-controller/pkg/config +k8s.io/kops/cmd/kops-controller/pkg/server k8s.io/kops/cmd/kube-apiserver-healthcheck k8s.io/kops/cmd/nodeup k8s.io/kops/dns-controller/cmd/dns-controller diff --git a/nodeup/pkg/model/BUILD.bazel b/nodeup/pkg/model/BUILD.bazel index dbd483a3fa5ac..1c393a5055b55 100644 --- a/nodeup/pkg/model/BUILD.bazel +++ b/nodeup/pkg/model/BUILD.bazel @@ -4,6 +4,7 @@ go_library( name = "go_default_library", srcs = [ "architecture.go", + "bootstrap_client.go", "cloudconfig.go", "containerd.go", "context.go", @@ -15,6 +16,7 @@ go_library( "file_assets.go", "firewall.go", "hooks.go", + "kops_controller.go", "kube_apiserver.go", "kube_apiserver_healthcheck.go", "kube_controller_manager.go", @@ -41,6 +43,7 @@ go_library( "//nodeup/pkg/distros:go_default_library", "//nodeup/pkg/model/resources:go_default_library", "//pkg/apis/kops:go_default_library", + "//pkg/apis/kops/model:go_default_library", "//pkg/apis/kops/util:go_default_library", "//pkg/apis/nodeup:go_default_library", "//pkg/assets:go_default_library", @@ -86,6 +89,7 @@ go_test( "containerd_test.go", "docker_test.go", "fakes_test.go", + "kops_controller_test.go", "kube_apiserver_test.go", "kube_controller_manager_test.go", "kube_proxy_test.go", diff --git a/nodeup/pkg/model/bootstrap_client.go b/nodeup/pkg/model/bootstrap_client.go new file mode 100644 index 0000000000000..f3daf2bd4660d --- /dev/null +++ b/nodeup/pkg/model/bootstrap_client.go @@ -0,0 +1,74 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package model + +import ( + "fmt" + + "k8s.io/kops/pkg/apis/kops" + "k8s.io/kops/upup/pkg/fi" + "k8s.io/kops/upup/pkg/fi/cloudup/awsup" + "k8s.io/kops/upup/pkg/fi/nodeup/nodetasks" +) + +// BootstrapClientBuilder calls kops-controller to bootstrap the node. +type BootstrapClientBuilder struct { + *NodeupModelContext +} + +func (b BootstrapClientBuilder) Build(c *fi.ModelBuilderContext) error { + if b.IsMaster || !b.UseKopsControllerForNodeBootstrap() { + return nil + } + + var authenticator fi.Authenticator + var err error + switch kops.CloudProviderID(b.Cluster.Spec.CloudProvider) { + case kops.CloudProviderAWS: + region, regionErr := awsup.FindRegion(b.Cluster) + if regionErr != nil { + return fmt.Errorf("querying AWS region: %v", regionErr) + } + authenticator, err = awsup.NewAWSAuthenticator(region) + default: + return fmt.Errorf("unsupported cloud provider %s", b.Cluster.Spec.CloudProvider) + } + if err != nil { + return err + } + + cert, err := b.GetCert(fi.CertificateIDCA) + if err != nil { + return err + } + + bootstrapClient := &nodetasks.BootstrapClient{ + Authenticator: authenticator, + CA: cert, + Certs: b.bootstrapCerts, + } + + for _, cert := range b.bootstrapCerts { + cert.Cert.Task = bootstrapClient + cert.Key.Task = bootstrapClient + } + + c.AddTask(bootstrapClient) + return nil +} + +var _ fi.ModelBuilder = &BootstrapClientBuilder{} diff --git a/nodeup/pkg/model/context.go b/nodeup/pkg/model/context.go index ecead295987a3..5733cfaba185f 100644 --- a/nodeup/pkg/model/context.go +++ b/nodeup/pkg/model/context.go @@ -25,6 +25,7 @@ import ( "k8s.io/klog" "k8s.io/kops/nodeup/pkg/distros" "k8s.io/kops/pkg/apis/kops" + "k8s.io/kops/pkg/apis/kops/model" "k8s.io/kops/pkg/apis/kops/util" "k8s.io/kops/pkg/apis/nodeup" "k8s.io/kops/pkg/systemd" @@ -56,6 +57,7 @@ type NodeupModelContext struct { IsMaster bool kubernetesVersion semver.Version + bootstrapCerts map[string]*nodetasks.BootstrapCert } // Init completes initialization of the object, for example pre-parsing the kubernetes version @@ -65,6 +67,7 @@ func (c *NodeupModelContext) Init() error { return fmt.Errorf("unable to parse KubernetesVersion %q", c.Cluster.Spec.KubernetesVersion) } c.kubernetesVersion = *k8sVersion + c.bootstrapCerts = map[string]*nodetasks.BootstrapCert{} if c.NodeupConfig.InstanceGroupRole == kops.InstanceGroupRoleMaster { c.IsMaster = true @@ -217,6 +220,51 @@ func (c *NodeupModelContext) BuildIssuedKubeconfig(name string, subject nodetask return kubeConfig.GetConfig() } +// BuildBootstrapKubeconfig generates a kubeconfig with a client certificate from either kops-controller or the state store. +func (c *NodeupModelContext) BuildBootstrapKubeconfig(name string, ctx *fi.ModelBuilderContext) (fi.Resource, error) { + if c.UseKopsControllerForNodeBootstrap() { + b, ok := c.bootstrapCerts[name] + if !ok { + b = &nodetasks.BootstrapCert{ + Cert: &fi.TaskDependentResource{}, + Key: &fi.TaskDependentResource{}, + } + c.bootstrapCerts[name] = b + } + + ca, err := c.GetCert(fi.CertificateIDCA) + if err != nil { + return nil, err + } + + kubeConfig := &nodetasks.KubeConfig{ + Name: name, + Cert: b.Cert, + Key: b.Key, + CA: fi.NewBytesResource(ca), + } + if c.IsMaster { + // @note: use https even for local connections, so we can turn off the insecure port + kubeConfig.ServerURL = "https://127.0.0.1" + } else { + kubeConfig.ServerURL = "https://" + c.Cluster.Spec.MasterInternalName + } + + err = ctx.EnsureTask(kubeConfig) + if err != nil { + return nil, err + } + + return kubeConfig.GetConfig(), nil + } else { + config, err := c.BuildPKIKubeconfig(name) + if err != nil { + return nil, err + } + return fi.NewStringResource(config), nil + } +} + // BuildPKIKubeconfig generates a kubeconfig func (c *NodeupModelContext) BuildPKIKubeconfig(name string) (string, error) { ca, err := c.GetCert(fi.CertificateIDCA) @@ -326,6 +374,11 @@ func (c *NodeupModelContext) UseEtcdTLSAuth() bool { return false } +// UseKopsControllerForNodeBootstrap checks if nodeup should use kops-controller to bootstrap. +func (c *NodeupModelContext) UseKopsControllerForNodeBootstrap() bool { + return model.UseKopsControllerForNodeBootstrap(c.Cluster) +} + // UseNodeAuthorization checks if have a node authorization policy func (c *NodeupModelContext) UseNodeAuthorization() bool { return c.Cluster.Spec.NodeAuthorization != nil @@ -372,20 +425,20 @@ func (c *NodeupModelContext) KubectlPath() string { return kubeletCommand } -// BuildCertificatePairTask creates the tasks to pull down the certificate and private key -func (c *NodeupModelContext) BuildCertificatePairTask(ctx *fi.ModelBuilderContext, key, path, filename string) error { +// BuildCertificatePairTask creates the tasks to create the certificate and private key files. +func (c *NodeupModelContext) BuildCertificatePairTask(ctx *fi.ModelBuilderContext, key, path, filename string, owner *string) error { certificateName := filepath.Join(path, filename+".pem") keyName := filepath.Join(path, filename+"-key.pem") - if err := c.BuildCertificateTask(ctx, key, certificateName); err != nil { + if err := c.BuildCertificateTask(ctx, key, certificateName, owner); err != nil { return err } - return c.BuildPrivateKeyTask(ctx, key, keyName) + return c.BuildPrivateKeyTask(ctx, key, keyName, owner) } -// BuildCertificateTask is responsible for build a certificate request task -func (c *NodeupModelContext) BuildCertificateTask(ctx *fi.ModelBuilderContext, name, filename string) error { +// BuildCertificateTask builds a task to create a certificate file. +func (c *NodeupModelContext) BuildCertificateTask(ctx *fi.ModelBuilderContext, name, filename string, owner *string) error { cert, err := c.KeyStore.FindCert(name) if err != nil { return err @@ -410,13 +463,14 @@ func (c *NodeupModelContext) BuildCertificateTask(ctx *fi.ModelBuilderContext, n Contents: fi.NewStringResource(serialized), Type: nodetasks.FileType_File, Mode: s("0600"), + Owner: owner, }) return nil } -// BuildPrivateKeyTask is responsible for build a certificate request task -func (c *NodeupModelContext) BuildPrivateKeyTask(ctx *fi.ModelBuilderContext, name, filename string) error { +// BuildPrivateKeyTask builds a task to create a private key file. +func (c *NodeupModelContext) BuildPrivateKeyTask(ctx *fi.ModelBuilderContext, name, filename string, owner *string) error { cert, err := c.KeyStore.FindPrivateKey(name) if err != nil { return err @@ -441,6 +495,7 @@ func (c *NodeupModelContext) BuildPrivateKeyTask(ctx *fi.ModelBuilderContext, na Contents: fi.NewStringResource(serialized), Type: nodetasks.FileType_File, Mode: s("0600"), + Owner: owner, }) return nil diff --git a/nodeup/pkg/model/etcd_manager_tls.go b/nodeup/pkg/model/etcd_manager_tls.go index 269334334f6e1..1388281a4001e 100644 --- a/nodeup/pkg/model/etcd_manager_tls.go +++ b/nodeup/pkg/model/etcd_manager_tls.go @@ -54,10 +54,10 @@ func (b *EtcdManagerTLSBuilder) Build(ctx *fi.ModelBuilderContext) error { return fmt.Errorf("keypair %q not found", keystoreName) } - if err := b.BuildCertificateTask(ctx, keystoreName, d+"/"+fileName+".crt"); err != nil { + if err := b.BuildCertificateTask(ctx, keystoreName, d+"/"+fileName+".crt", nil); err != nil { return err } - if err := b.BuildPrivateKeyTask(ctx, keystoreName, d+"/"+fileName+".key"); err != nil { + if err := b.BuildPrivateKeyTask(ctx, keystoreName, d+"/"+fileName+".key", nil); err != nil { return err } } diff --git a/nodeup/pkg/model/kops_controller.go b/nodeup/pkg/model/kops_controller.go new file mode 100644 index 0000000000000..d8c6f4789eee0 --- /dev/null +++ b/nodeup/pkg/model/kops_controller.go @@ -0,0 +1,93 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package model + +import ( + "path/filepath" + + "k8s.io/kops/pkg/wellknownusers" + "k8s.io/kops/upup/pkg/fi" + "k8s.io/kops/upup/pkg/fi/nodeup/nodetasks" +) + +// KopsControllerBuilder installs the keys for a kops-controller. +type KopsControllerBuilder struct { + *NodeupModelContext +} + +var _ fi.ModelBuilder = &KopsControllerBuilder{} + +// Build is responsible for configuring keys that will be used by kops-controller (via hostPath) +func (b *KopsControllerBuilder) Build(c *fi.ModelBuilderContext) error { + if !b.IsMaster { + return nil + } + + // Create the directory, even if we aren't going to populate it + pkiDir := "/etc/kubernetes/kops-controller" + c.AddTask(&nodetasks.File{ + Path: pkiDir, + Type: nodetasks.FileType_Directory, + Mode: s("0755"), + }) + + if !b.UseKopsControllerForNodeBootstrap() { + return nil + } + + // We run kops-controller under an unprivileged user (wellknownusers.KopsControllerID), and then grant specific permissions + c.AddTask(&nodetasks.UserTask{ + Name: wellknownusers.KopsControllerName, + UID: wellknownusers.KopsControllerID, + Shell: "/sbin/nologin", + }) + + issueCert := &nodetasks.IssueCert{ + Name: "kops-controller", + Signer: fi.CertificateIDCA, + Type: "server", + Subject: nodetasks.PKIXName{CommonName: "kops-controller"}, + AlternateNames: []string{b.Cluster.Spec.MasterInternalName}, + } + c.AddTask(issueCert) + + certResource, keyResource, _ := issueCert.GetResources() + c.AddTask(&nodetasks.File{ + Path: filepath.Join(pkiDir, "kops-controller.crt"), + Contents: certResource, + Type: nodetasks.FileType_File, + Mode: s("0644"), + Owner: s(wellknownusers.KopsControllerName), + }) + c.AddTask(&nodetasks.File{ + Path: filepath.Join(pkiDir, "kops-controller.key"), + Contents: keyResource, + Type: nodetasks.FileType_File, + Mode: s("0600"), + Owner: s(wellknownusers.KopsControllerName), + }) + + for _, cert := range []string{fi.CertificateIDCA} { + owner := wellknownusers.KopsControllerName + err := b.BuildCertificatePairTask(c, cert, pkiDir, cert, &owner) + if err != nil { + return err + } + } + + return nil +} diff --git a/nodeup/pkg/model/kops_controller_test.go b/nodeup/pkg/model/kops_controller_test.go new file mode 100644 index 0000000000000..46c64e96428a3 --- /dev/null +++ b/nodeup/pkg/model/kops_controller_test.go @@ -0,0 +1,30 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package model + +import ( + "testing" + + "k8s.io/kops/upup/pkg/fi" +) + +func TestKopsControllerBuilder(t *testing.T) { + RunGoldenTest(t, "tests/golden/minimal", "kops-controller", func(nodeupModelContext *NodeupModelContext, target *fi.ModelBuilderContext) error { + builder := KopsControllerBuilder{NodeupModelContext: nodeupModelContext} + return builder.Build(target) + }) +} diff --git a/nodeup/pkg/model/kube_controller_manager.go b/nodeup/pkg/model/kube_controller_manager.go index 174084bfb7dcc..d61b3274715f0 100644 --- a/nodeup/pkg/model/kube_controller_manager.go +++ b/nodeup/pkg/model/kube_controller_manager.go @@ -53,7 +53,7 @@ func (b *KubeControllerManagerBuilder) Build(c *fi.ModelBuilderContext) error { // Include the CA Key // @TODO: use a per-machine key? use KMS? - if err := b.BuildPrivateKeyTask(c, fi.CertificateIDCA, "ca.key"); err != nil { + if err := b.BuildPrivateKeyTask(c, fi.CertificateIDCA, "ca.key", nil); err != nil { return err } diff --git a/nodeup/pkg/model/kubelet.go b/nodeup/pkg/model/kubelet.go index bf6d01788ae57..6bbaa44ad1a7d 100644 --- a/nodeup/pkg/model/kubelet.go +++ b/nodeup/pkg/model/kubelet.go @@ -92,7 +92,10 @@ func (b *KubeletBuilder) Build(c *fi.ModelBuilderContext) error { if err != nil { return err } - c.EnsureTask(t) + err = c.EnsureTask(t) + if err != nil { + return err + } } } { @@ -103,26 +106,20 @@ func (b *KubeletBuilder) Build(c *fi.ModelBuilderContext) error { Mode: s("0755"), }) - // @check if bootstrap tokens are enabled and create the appropreiate certificates - if b.UseBootstrapTokens() { - // @check if a master and if so, we bypass the token strapping and instead generate our own kubeconfig - if b.IsMaster { - klog.V(3).Info("kubelet bootstrap tokens are enabled and running on a master") - - err := b.buildMasterKubeletKubeconfig(c) - if err != nil { - return err - } + if b.IsMaster || !b.UseBootstrapTokens() { + var kubeconfig fi.Resource + if b.IsMaster && (b.IsKubernetesGTE("1.19") || b.UseBootstrapTokens()) { + kubeconfig, err = b.buildMasterKubeletKubeconfig(c) + } else { + kubeconfig, err = b.BuildBootstrapKubeconfig("kubelet", c) } - } else { - kubeconfig, err := b.BuildPKIKubeconfig("kubelet") if err != nil { return err } c.AddTask(&nodetasks.File{ Path: b.KubeletKubeConfig(), - Contents: fi.NewStringResource(kubeconfig), + Contents: kubeconfig, Type: nodetasks.FileType_File, Mode: s("0400"), }) @@ -525,23 +522,15 @@ func (b *KubeletBuilder) buildKubeletConfigSpec() (*kops.KubeletConfigSpec, erro } // buildMasterKubeletKubeconfig builds a kubeconfig for the master kubelet, self-signing the kubelet cert -func (b *KubeletBuilder) buildMasterKubeletKubeconfig(c *fi.ModelBuilderContext) error { +func (b *KubeletBuilder) buildMasterKubeletKubeconfig(c *fi.ModelBuilderContext) (fi.Resource, error) { nodeName, err := b.NodeName() if err != nil { - return fmt.Errorf("error getting NodeName: %v", err) + return nil, fmt.Errorf("error getting NodeName: %v", err) } certName := nodetasks.PKIXName{ CommonName: fmt.Sprintf("system:node:%s", nodeName), Organization: []string{rbac.NodesGroup}, } - kubeconfig := b.BuildIssuedKubeconfig("kubelet", certName, c) - c.AddTask(&nodetasks.File{ - Path: b.KubeletKubeConfig(), - Contents: kubeconfig, - Type: nodetasks.FileType_File, - Mode: s("600"), - }) - - return nil + return b.BuildIssuedKubeconfig("kubelet", certName, c), nil } diff --git a/nodeup/pkg/model/networking/calico.go b/nodeup/pkg/model/networking/calico.go index f50e67e9696f4..1f682fd540716 100644 --- a/nodeup/pkg/model/networking/calico.go +++ b/nodeup/pkg/model/networking/calico.go @@ -46,13 +46,13 @@ func (b *CalicoBuilder) Build(c *fi.ModelBuilderContext) error { certificate := filepath.Join(dirname, name+".pem") key := filepath.Join(dirname, name+"-key.pem") - if err := b.BuildCertificateTask(c, name, certificate); err != nil { + if err := b.BuildCertificateTask(c, name, certificate, nil); err != nil { return err } - if err := b.BuildPrivateKeyTask(c, name, key); err != nil { + if err := b.BuildPrivateKeyTask(c, name, key, nil); err != nil { return err } - if err := b.BuildCertificateTask(c, fi.CertificateIDCA, ca); err != nil { + if err := b.BuildCertificateTask(c, fi.CertificateIDCA, ca, nil); err != nil { return err } } diff --git a/nodeup/pkg/model/networking/cilium.go b/nodeup/pkg/model/networking/cilium.go index c3c243d651107..a2585732059f5 100644 --- a/nodeup/pkg/model/networking/cilium.go +++ b/nodeup/pkg/model/networking/cilium.go @@ -123,10 +123,10 @@ func (b *CiliumBuilder) buildCiliumEtcdSecrets(c *fi.ModelBuilderContext) error return err } - if err := b.BuildCertificateTask(c, keystoreName, d+"/"+fileName+".crt"); err != nil { + if err := b.BuildCertificateTask(c, keystoreName, d+"/"+fileName+".crt", nil); err != nil { return err } - if err := b.BuildPrivateKeyTask(c, keystoreName, d+"/"+fileName+".key"); err != nil { + if err := b.BuildPrivateKeyTask(c, keystoreName, d+"/"+fileName+".key", nil); err != nil { return err } } diff --git a/nodeup/pkg/model/node_authorizer.go b/nodeup/pkg/model/node_authorizer.go index e729c8e1bda9e..a3fdfa546a501 100644 --- a/nodeup/pkg/model/node_authorizer.go +++ b/nodeup/pkg/model/node_authorizer.go @@ -43,11 +43,11 @@ func (b *NodeAuthorizationBuilder) Build(c *fi.ModelBuilderContext) error { if b.UseBootstrapTokens() && b.IsMaster { name := "node-authorizer" // creates /src/kubernetes/node-authorizer/{tls,tls-key}.pem - if err := b.BuildCertificatePairTask(c, name, name, "tls"); err != nil { + if err := b.BuildCertificatePairTask(c, name, name, "tls", nil); err != nil { return err } // creates /src/kubernetes/node-authorizer/ca.pem - if err := b.BuildCertificateTask(c, fi.CertificateIDCA, filepath.Join(name, "ca.pem")); err != nil { + if err := b.BuildCertificateTask(c, fi.CertificateIDCA, filepath.Join(name, "ca.pem"), nil); err != nil { return err } } @@ -55,10 +55,10 @@ func (b *NodeAuthorizationBuilder) Build(c *fi.ModelBuilderContext) error { authorizerDir := "node-authorizer" // @check if bootstrap tokens are enabled and download client certificates for nodes if b.UseBootstrapTokens() && !b.IsMaster { - if err := b.BuildCertificatePairTask(c, "node-authorizer-client", authorizerDir, "tls"); err != nil { + if err := b.BuildCertificatePairTask(c, "node-authorizer-client", authorizerDir, "tls", nil); err != nil { return err } - if err := b.BuildCertificateTask(c, fi.CertificateIDCA, authorizerDir+"/ca.pem"); err != nil { + if err := b.BuildCertificateTask(c, fi.CertificateIDCA, authorizerDir+"/ca.pem", nil); err != nil { return err } } diff --git a/nodeup/pkg/model/protokube.go b/nodeup/pkg/model/protokube.go index 1a605c49ae479..ff6daac4fbe04 100644 --- a/nodeup/pkg/model/protokube.go +++ b/nodeup/pkg/model/protokube.go @@ -82,12 +82,12 @@ func (t *ProtokubeBuilder) Build(c *fi.ModelBuilderContext) error { // retrieve the etcd peer certificates and private keys from the keystore if !t.UseEtcdManager() && t.UseEtcdTLS() { for _, x := range []string{"etcd", "etcd-peer", "etcd-client"} { - if err := t.BuildCertificateTask(c, x, fmt.Sprintf("%s.pem", x)); err != nil { + if err := t.BuildCertificateTask(c, x, fmt.Sprintf("%s.pem", x), nil); err != nil { return err } } for _, x := range []string{"etcd", "etcd-peer", "etcd-client"} { - if err := t.BuildPrivateKeyTask(c, x, fmt.Sprintf("%s-key.pem", x)); err != nil { + if err := t.BuildPrivateKeyTask(c, x, fmt.Sprintf("%s-key.pem", x), nil); err != nil { return err } } diff --git a/nodeup/pkg/model/secrets.go b/nodeup/pkg/model/secrets.go index 90985ce379033..7ed566814a8b5 100644 --- a/nodeup/pkg/model/secrets.go +++ b/nodeup/pkg/model/secrets.go @@ -50,7 +50,7 @@ func (b *SecretBuilder) Build(c *fi.ModelBuilderContext) error { } // @step: retrieve the platform ca - if err := b.BuildCertificateTask(c, fi.CertificateIDCA, "ca.crt"); err != nil { + if err := b.BuildCertificateTask(c, fi.CertificateIDCA, "ca.crt", nil); err != nil { return err } @@ -142,7 +142,7 @@ func (b *SecretBuilder) Build(c *fi.ModelBuilderContext) error { } } - if err := b.BuildPrivateKeyTask(c, "master", "service-account.key"); err != nil { + if err := b.BuildPrivateKeyTask(c, "master", "service-account.key", nil); err != nil { return err } diff --git a/nodeup/pkg/model/tests/golden/minimal/tasks-kops-controller.yaml b/nodeup/pkg/model/tests/golden/minimal/tasks-kops-controller.yaml new file mode 100644 index 0000000000000..8e5f9eb837d6d --- /dev/null +++ b/nodeup/pkg/model/tests/golden/minimal/tasks-kops-controller.yaml @@ -0,0 +1,3 @@ +mode: "0755" +path: /etc/kubernetes/kops-controller +type: directory diff --git a/pkg/apis/kops/model/BUILD.bazel b/pkg/apis/kops/model/BUILD.bazel index 395598832bfb3..dd9a0e9503d7c 100644 --- a/pkg/apis/kops/model/BUILD.bazel +++ b/pkg/apis/kops/model/BUILD.bazel @@ -2,7 +2,10 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") go_library( name = "go_default_library", - srcs = ["utils.go"], + srcs = [ + "features.go", + "utils.go", + ], importpath = "k8s.io/kops/pkg/apis/kops/model", visibility = ["//visibility:public"], deps = [ diff --git a/pkg/apis/kops/model/features.go b/pkg/apis/kops/model/features.go new file mode 100644 index 0000000000000..fbc0836986d82 --- /dev/null +++ b/pkg/apis/kops/model/features.go @@ -0,0 +1,26 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package model + +import ( + "k8s.io/kops/pkg/apis/kops" +) + +// UseKopsControllerForNodeBootstrap is true if nodeup should use kops-controller for bootstrapping. +func UseKopsControllerForNodeBootstrap(cluster *kops.Cluster) bool { + return kops.CloudProviderID(cluster.Spec.CloudProvider) == kops.CloudProviderAWS && cluster.IsKubernetesGTE("1.19") +} diff --git a/pkg/apis/nodeup/BUILD.bazel b/pkg/apis/nodeup/BUILD.bazel index 36622381b987b..21643fe611128 100644 --- a/pkg/apis/nodeup/BUILD.bazel +++ b/pkg/apis/nodeup/BUILD.bazel @@ -2,7 +2,10 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library") go_library( name = "go_default_library", - srcs = ["config.go"], + srcs = [ + "bootstrap.go", + "config.go", + ], importpath = "k8s.io/kops/pkg/apis/nodeup", visibility = ["//visibility:public"], deps = [ diff --git a/pkg/apis/nodeup/bootstrap.go b/pkg/apis/nodeup/bootstrap.go new file mode 100644 index 0000000000000..095718233d77b --- /dev/null +++ b/pkg/apis/nodeup/bootstrap.go @@ -0,0 +1,33 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package nodeup + +const BootstrapAPIVersion = "bootstrap.kops.k8s.io/v1alpha1" + +// BootstrapRequest is a request from nodeup to kops-controller for bootstrapping a node. +type BootstrapRequest struct { + // APIVersion defines the versioned schema of this representation of a request. + APIVersion string `json:"apiVersion"` + // Certs are the requested certificates and their respective public keys. + Certs map[string]string `json:"certs"` +} + +// BootstrapRespose is a response to a BootstrapRequest. +type BootstrapResponse struct { + // Certs are the issued certificates. + Certs map[string]string +} diff --git a/pkg/model/components/apiserver.go b/pkg/model/components/apiserver.go index 3e960d10fe256..f290d25abc043 100644 --- a/pkg/model/components/apiserver.go +++ b/pkg/model/components/apiserver.go @@ -85,8 +85,9 @@ func (b *KubeAPIServerOptionsBuilder) BuildOptions(o interface{}) error { } else if clusterSpec.Authorization.RBAC != nil { var modes []string - if fi.BoolValue(clusterSpec.KubeAPIServer.EnableBootstrapAuthToken) { + if b.IsKubernetesGTE("1.19") || fi.BoolValue(clusterSpec.KubeAPIServer.EnableBootstrapAuthToken) { // Enable the Node authorizer, used for special per-node RBAC policies + // Enable by default from 1.19 - it's an important part of limiting blast radius modes = append(modes, "Node") } modes = append(modes, "RBAC") diff --git a/pkg/model/context.go b/pkg/model/context.go index e3d9d26341ca4..e51f28de7d707 100644 --- a/pkg/model/context.go +++ b/pkg/model/context.go @@ -267,9 +267,14 @@ func (m *KopsModelContext) CloudTags(name string, shared bool) map[string]string return tags } +// UseKopsControllerForNodeBootstrap checks if nodeup should use kops-controller to bootstrap. +func (m *KopsModelContext) UseKopsControllerForNodeBootstrap() bool { + return model.UseKopsControllerForNodeBootstrap(m.Cluster) +} + // UseBootstrapTokens checks if bootstrap tokens are enabled func (m *KopsModelContext) UseBootstrapTokens() bool { - if m.Cluster.Spec.KubeAPIServer == nil { + if m.Cluster.Spec.KubeAPIServer == nil || m.UseKopsControllerForNodeBootstrap() { return false } diff --git a/pkg/model/iam/BUILD.bazel b/pkg/model/iam/BUILD.bazel index 96e657ca95aa0..dd2060d457e4f 100644 --- a/pkg/model/iam/BUILD.bazel +++ b/pkg/model/iam/BUILD.bazel @@ -10,6 +10,7 @@ go_library( visibility = ["//visibility:public"], deps = [ "//pkg/apis/kops:go_default_library", + "//pkg/apis/kops/model:go_default_library", "//pkg/util/stringorslice:go_default_library", "//upup/pkg/fi:go_default_library", "//upup/pkg/fi/cloudup/awstasks:go_default_library", diff --git a/pkg/model/iam/iam_builder.go b/pkg/model/iam/iam_builder.go index 6ed0c1b9f4ad3..aa58a252d89ef 100644 --- a/pkg/model/iam/iam_builder.go +++ b/pkg/model/iam/iam_builder.go @@ -35,6 +35,7 @@ import ( "k8s.io/apimachinery/pkg/util/sets" "k8s.io/klog" + "k8s.io/kops/pkg/apis/kops/model" "k8s.io/kops/pkg/apis/kops" "k8s.io/kops/pkg/util/stringorslice" @@ -458,7 +459,9 @@ func ReadableStatePaths(cluster *kops.Cluster, role kops.InstanceGroupRole) ([]s ) // @check if bootstrap tokens are enabled and if so enable access to client certificate - if UseBootstrapTokens(cluster) { + if model.UseKopsControllerForNodeBootstrap(cluster) { + // no additional permissions + } else if useBootstrapTokens(cluster) { paths = append(paths, "/pki/private/node-authorizer-client/*") } else { paths = append(paths, "/pki/private/kubelet/*") @@ -539,9 +542,9 @@ func (b *PolicyResource) Open() (io.Reader, error) { return bytes.NewReader([]byte(j)), nil } -// UseBootstrapTokens check if we are using bootstrap tokens - @TODO, i don't like this we should probably pass in +// useBootstrapTokens check if we are using bootstrap tokens - @TODO, i don't like this we should probably pass in // the kops model into the builder rather than duplicating the code. I'll leave for another PR -func UseBootstrapTokens(cluster *kops.Cluster) bool { +func useBootstrapTokens(cluster *kops.Cluster) bool { if cluster.Spec.KubeAPIServer == nil { return false } diff --git a/pkg/model/pki.go b/pkg/model/pki.go index e3a954e09a006..db9a09e6c6277 100644 --- a/pkg/model/pki.go +++ b/pkg/model/pki.go @@ -48,9 +48,9 @@ func (b *PKIModelBuilder) Build(c *fi.ModelBuilderContext) error { c.AddTask(defaultCA) { - // @check of bootstrap tokens are enable if so, disable the creation of the kubelet certificate - we also + // @check if kops-controller bootstrap or bootstrap tokens are enabled. If so, disable the creation of the kubelet certificate - we also // block at the IAM level for AWS cluster for pre-existing clusters. - if !b.UseBootstrapTokens() { + if !b.UseKopsControllerForNodeBootstrap() && !b.UseBootstrapTokens() { c.AddTask(&fitasks.Keypair{ Name: fi.String("kubelet"), Lifecycle: b.Lifecycle, diff --git a/pkg/pki/issue.go b/pkg/pki/issue.go index ccf4238973a09..a35e00b3170b5 100644 --- a/pkg/pki/issue.go +++ b/pkg/pki/issue.go @@ -17,6 +17,7 @@ limitations under the License. package pki import ( + "crypto" "crypto/x509" "crypto/x509/pkix" "fmt" @@ -43,7 +44,9 @@ type IssueCertRequest struct { // AlternateNames is a list of alternative names for this certificate. AlternateNames []string - // PrivateKey is the private key for this certificate. If nil, a new private key will be generated. + // PublicKey is the public key for this certificate. If nil, it will be calculated from PrivateKey. + PublicKey crypto.PublicKey + // PrivateKey is the private key for this certificate. If both this and PublicKey are nil, a new private key will be generated. PrivateKey *PrivateKey // Validity is the certificate validity. The default is 10 years. Validity time.Duration @@ -130,7 +133,9 @@ func IssueCert(request *IssueCertRequest, keystore Keystore) (issuedCertificate } privateKey := request.PrivateKey - if privateKey == nil { + if request.PublicKey != nil { + template.PublicKey = request.PublicKey + } else if privateKey == nil { var err error privateKey, err = GeneratePrivateKey() if err != nil { diff --git a/pkg/wellknownports/wellknownports.go b/pkg/wellknownports/wellknownports.go index 71c31b7c790f5..03b7912aeacca 100644 --- a/pkg/wellknownports/wellknownports.go +++ b/pkg/wellknownports/wellknownports.go @@ -44,6 +44,9 @@ const ( // DNSControllerGossipMemberlist is the port where dns-controller listens for the memberlist-backed gossip DNSControllerGossipMemberlist = 3993 + // KopsControllerPort is the port where kops-controller listens. + KopsControllerPort = 3992 + // 4001 is etcd main, 4002 is etcd events, 4003 is etcd cilium // KubeAPIServerHealthCheck is the port where kube-apiserver-healthcheck listens. diff --git a/pkg/wellknownusers/wellknownusers.go b/pkg/wellknownusers/wellknownusers.go index 4e9580a4b6fd2..cc470873a6cd9 100644 --- a/pkg/wellknownusers/wellknownusers.go +++ b/pkg/wellknownusers/wellknownusers.go @@ -21,7 +21,7 @@ package wellknownusers const ( // Generic is the user id we use for non-privileged containers, where we don't need extra permissions - // Used by e.g. dns-controller, kops-controller + // Used by e.g. dns-controller Generic = 10001 // LegacyEtcd is the user id for the etcd user under the legacy provider @@ -30,6 +30,13 @@ const ( // AWSAuthenticator is the user-id for the aws-iam-authenticator (built externally) AWSAuthenticator = 10000 + // KopsControllerID is the user id for kops-controller, which needs some extra permissions e.g. to write local logs + // This should match the user in cmd/kops-controller/BUILD.bazel + KopsControllerID = 10011 + + // KopsControllerName is the username for the kops-controller user + KopsControllerName = "kops-controller" + // KubeApiserverHealthcheckID is the user id for kube-apiserver-healthcheck sidecar // The user needs some extra permissions e.g. to read local secrets // This should match the user in cmd/kube-apiserver-healthcheck/BUILD.bazel diff --git a/upup/models/bindata.go b/upup/models/bindata.go index ae4bb1e2dcc4f..b66390f731586 100644 --- a/upup/models/bindata.go +++ b/upup/models/bindata.go @@ -2536,8 +2536,10 @@ spec: name: etc-ssl-certs readOnly: true {{ end }} - - mountPath: /etc/kubernetes/kops-controller/ + - mountPath: /etc/kubernetes/kops-controller/config/ name: kops-controller-config + - mountPath: /etc/kubernetes/kops-controller/pki/ + name: kops-controller-pki command: {{ range $arg := KopsControllerArgv }} - "{{ $arg }}" @@ -2565,7 +2567,10 @@ spec: - name: kops-controller-config configMap: name: kops-controller - + - name: kops-controller-pki + hostPath: + path: /etc/kubernetes/kops-controller/ + type: Directory --- apiVersion: v1 diff --git a/upup/models/cloudup/resources/addons/kops-controller.addons.k8s.io/k8s-1.16.yaml.template b/upup/models/cloudup/resources/addons/kops-controller.addons.k8s.io/k8s-1.16.yaml.template index 5d4a91fd1c80a..4866f9302bca1 100644 --- a/upup/models/cloudup/resources/addons/kops-controller.addons.k8s.io/k8s-1.16.yaml.template +++ b/upup/models/cloudup/resources/addons/kops-controller.addons.k8s.io/k8s-1.16.yaml.template @@ -53,8 +53,10 @@ spec: name: etc-ssl-certs readOnly: true {{ end }} - - mountPath: /etc/kubernetes/kops-controller/ + - mountPath: /etc/kubernetes/kops-controller/config/ name: kops-controller-config + - mountPath: /etc/kubernetes/kops-controller/pki/ + name: kops-controller-pki command: {{ range $arg := KopsControllerArgv }} - "{{ $arg }}" @@ -82,7 +84,10 @@ spec: - name: kops-controller-config configMap: name: kops-controller - + - name: kops-controller-pki + hostPath: + path: /etc/kubernetes/kops-controller/ + type: Directory --- apiVersion: v1 diff --git a/upup/pkg/fi/BUILD.bazel b/upup/pkg/fi/BUILD.bazel index 021c74f44ae7e..f8a7e2db0b4ce 100644 --- a/upup/pkg/fi/BUILD.bazel +++ b/upup/pkg/fi/BUILD.bazel @@ -4,6 +4,7 @@ go_library( name = "go_default_library", srcs = [ "assetstore.go", + "authenticate.go", "ca.go", "changes.go", "clientset_castore.go", diff --git a/upup/pkg/fi/authenticate.go b/upup/pkg/fi/authenticate.go new file mode 100644 index 0000000000000..3edd2a1f271b6 --- /dev/null +++ b/upup/pkg/fi/authenticate.go @@ -0,0 +1,33 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package fi + +// Authenticator generates authentication credentials for requests. +type Authenticator interface { + CreateToken(body []byte) (string, error) +} + +// VerifyResult is the result of a successfully verified request. +type VerifyResult struct { + // Nodename is the name that this node is authorized to use. + NodeName string +} + +// Verifier verifies authentication credentials for requests. +type Verifier interface { + VerifyToken(token string, body []byte) (*VerifyResult, error) +} diff --git a/upup/pkg/fi/cloudup/awsup/BUILD.bazel b/upup/pkg/fi/cloudup/awsup/BUILD.bazel index c5e5d5a811b6c..4d3dca315b77b 100644 --- a/upup/pkg/fi/cloudup/awsup/BUILD.bazel +++ b/upup/pkg/fi/cloudup/awsup/BUILD.bazel @@ -4,8 +4,10 @@ go_library( name = "go_default_library", srcs = [ "aws_apitarget.go", + "aws_authenticator.go", "aws_cloud.go", "aws_utils.go", + "aws_verifier.go", "instancegroups.go", "logging_retryer.go", "machine_types.go", @@ -44,6 +46,7 @@ go_library( "//vendor/github.com/aws/aws-sdk-go/service/iam/iamiface:go_default_library", "//vendor/github.com/aws/aws-sdk-go/service/route53:go_default_library", "//vendor/github.com/aws/aws-sdk-go/service/route53/route53iface:go_default_library", + "//vendor/github.com/aws/aws-sdk-go/service/sts:go_default_library", "//vendor/k8s.io/api/core/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//vendor/k8s.io/klog:go_default_library", diff --git a/upup/pkg/fi/cloudup/awsup/aws_authenticator.go b/upup/pkg/fi/cloudup/awsup/aws_authenticator.go new file mode 100644 index 0000000000000..9a471331d5765 --- /dev/null +++ b/upup/pkg/fi/cloudup/awsup/aws_authenticator.go @@ -0,0 +1,64 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package awsup + +import ( + "crypto/sha256" + "encoding/base64" + "encoding/json" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/sts" + "k8s.io/kops/upup/pkg/fi" +) + +const AWSAuthenticationTokenPrefix = "x-aws-sts " + +type awsAuthenticator struct { + sts *sts.STS +} + +var _ fi.Authenticator = &awsAuthenticator{} + +func NewAWSAuthenticator(region string) (fi.Authenticator, error) { + config := aws.NewConfig().WithCredentialsChainVerboseErrors(true).WithRegion(region) + sess, err := session.NewSession(config) + if err != nil { + return nil, err + } + return &awsAuthenticator{ + sts: sts.New(sess), + }, nil +} + +func (a awsAuthenticator) CreateToken(body []byte) (string, error) { + sha := sha256.Sum256(body) + + stsRequest, _ := a.sts.GetCallerIdentityRequest(nil) + + // Ensure the signature is only valid for this particular body content. + stsRequest.HTTPRequest.Header.Add("X-Kops-Request-SHA", base64.RawStdEncoding.EncodeToString(sha[:])) + + err := stsRequest.Sign() + if err != nil { + return "", err + } + + headers, _ := json.Marshal(stsRequest.HTTPRequest.Header) + return AWSAuthenticationTokenPrefix + base64.StdEncoding.EncodeToString(headers), nil +} diff --git a/upup/pkg/fi/cloudup/awsup/aws_verifier.go b/upup/pkg/fi/cloudup/awsup/aws_verifier.go new file mode 100644 index 0000000000000..cf4e14e884294 --- /dev/null +++ b/upup/pkg/fi/cloudup/awsup/aws_verifier.go @@ -0,0 +1,231 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package awsup + +import ( + "bytes" + "crypto/sha256" + "encoding/base64" + "encoding/json" + "encoding/xml" + "fmt" + "io" + "io/ioutil" + "net" + "net/http" + "strconv" + "strings" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/aws/aws-sdk-go/service/sts" + "k8s.io/kops/upup/pkg/fi" +) + +type AWSVerifierOptions struct { + // NodesRoles are the IAM roles that worker nodes are permitted to have. + NodesRoles []string `json:"nodesRoles"` + // Region is the AWS region of the cluster. + Region string +} + +type awsVerifier struct { + accountId string + partition string + opt AWSVerifierOptions + + ec2 *ec2.EC2 + sts *sts.STS + client http.Client +} + +var _ fi.Verifier = &awsVerifier{} + +func NewAWSVerifier(opt *AWSVerifierOptions) (fi.Verifier, error) { + config := aws.NewConfig().WithCredentialsChainVerboseErrors(true).WithRegion(opt.Region) + sess, err := session.NewSession(config) + if err != nil { + return nil, err + } + + stsClient := sts.New(sess) + identity, err := stsClient.GetCallerIdentity(&sts.GetCallerIdentityInput{}) + if err != nil { + return nil, err + } + + partition := strings.Split(aws.StringValue(identity.Arn), ":")[1] + + ec2Client := ec2.New(sess, config) + + return &awsVerifier{ + accountId: aws.StringValue(identity.Account), + partition: partition, + opt: *opt, + ec2: ec2Client, + sts: stsClient, + client: http.Client{ + Transport: &http.Transport{ + Proxy: http.ProxyFromEnvironment, + DialContext: (&net.Dialer{ + Timeout: 30 * time.Second, + }).DialContext, + ForceAttemptHTTP2: true, + DisableKeepAlives: true, + MaxIdleConnsPerHost: -1, + TLSHandshakeTimeout: 10 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + }, + CheckRedirect: func(req *http.Request, via []*http.Request) error { + return http.ErrUseLastResponse + }, + }, + }, nil +} + +type GetCallerIdentityResponse struct { + XMLName xml.Name `xml:"GetCallerIdentityResponse"` + GetCallerIdentityResult []GetCallerIdentityResult `xml:"GetCallerIdentityResult"` + ResponseMetadata []ResponseMetadata `xml:"ResponseMetadata"` +} + +type GetCallerIdentityResult struct { + Arn string `xml:"Arn"` + UserId string `xml:"UserId"` + Account string `xml:"Account"` +} + +type ResponseMetadata struct { + RequestId string `xml:"RequestId"` +} + +func (a awsVerifier) VerifyToken(token string, body []byte) (*fi.VerifyResult, error) { + if !strings.HasPrefix(token, AWSAuthenticationTokenPrefix) { + return nil, fmt.Errorf("incorrect authorization type") + } + token = strings.TrimPrefix(token, AWSAuthenticationTokenPrefix) + + // We rely on the client and server using the same version of the same STS library. + stsRequest, _ := a.sts.GetCallerIdentityRequest(nil) + err := stsRequest.Sign() + if err != nil { + return nil, fmt.Errorf("creating identity request: %v", err) + } + + stsRequest.HTTPRequest.Header = nil + tokenBytes, err := base64.StdEncoding.DecodeString(token) + if err != nil { + return nil, fmt.Errorf("decoding authorization token: %v", err) + } + err = json.Unmarshal(tokenBytes, &stsRequest.HTTPRequest.Header) + if err != nil { + return nil, fmt.Errorf("unmarshalling authorization token: %v", err) + } + + // Verify the token has signed the body content. + sha := sha256.Sum256(body) + if stsRequest.HTTPRequest.Header.Get("X-Kops-Request-SHA") != base64.RawStdEncoding.EncodeToString(sha[:]) { + return nil, fmt.Errorf("incorrect SHA") + } + + requestBytes, _ := ioutil.ReadAll(stsRequest.Body) + _, _ = stsRequest.Body.Seek(0, io.SeekStart) + if stsRequest.HTTPRequest.Header.Get("Content-Length") != strconv.Itoa(len(requestBytes)) { + return nil, fmt.Errorf("incorrect content-length") + } + + response, err := a.client.Do(stsRequest.HTTPRequest) + if err != nil { + return nil, fmt.Errorf("sending STS request: %v", err) + } + if response != nil { + defer response.Body.Close() + } + + responseBody, err := ioutil.ReadAll(response.Body) + if err != nil { + return nil, fmt.Errorf("reading STS response: %v", err) + } + if response.StatusCode != 200 { + return nil, fmt.Errorf("received status code %d from STS: %s", response.StatusCode, string(responseBody)) + } + + result := GetCallerIdentityResponse{} + err = xml.NewDecoder(bytes.NewReader(responseBody)).Decode(&result) + if err != nil { + return nil, fmt.Errorf("decoding STS response: %v", err) + } + + if result.GetCallerIdentityResult[0].Account != a.accountId { + return nil, fmt.Errorf("incorrect account %s", result.GetCallerIdentityResult[0].Account) + } + + arn := result.GetCallerIdentityResult[0].Arn + parts := strings.Split(arn, ":") + if len(parts) != 6 { + return nil, fmt.Errorf("arn %q contains unexpected number of colons", arn) + } + if parts[0] != "arn" { + return nil, fmt.Errorf("arn %q doesn't start with \"arn:\"", arn) + } + if parts[1] != a.partition { + return nil, fmt.Errorf("arn %q not in partion %q", arn, a.partition) + } + if parts[2] != "iam" && parts[2] != "sts" { + return nil, fmt.Errorf("arn %q has unrecognized service", arn) + } + // parts[3] is region + // parts[4] is account + resource := strings.Split(parts[5], "/") + if resource[0] != "assumed-role" { + return nil, fmt.Errorf("arn %q has unrecognized type", arn) + } + if len(resource) < 3 { + return nil, fmt.Errorf("arn %q contains too few slashes", arn) + } + found := false + for _, role := range a.opt.NodesRoles { + if resource[1] == role { + found = true + break + } + } + if !found { + return nil, fmt.Errorf("arn %q does not contain acceptable node role", arn) + } + + instanceID := resource[2] + instances, err := a.ec2.DescribeInstances(&ec2.DescribeInstancesInput{ + InstanceIds: aws.StringSlice([]string{instanceID}), + }) + if err != nil { + return nil, fmt.Errorf("describing instance for arn %q", arn) + } + + if len(instances.Reservations) <= 0 || len(instances.Reservations[0].Instances) <= 0 { + return nil, fmt.Errorf("missing instance id: %s", instanceID) + } + if len(instances.Reservations[0].Instances) > 1 { + return nil, fmt.Errorf("found multiple instances with instance id: %s", instanceID) + } + + return &fi.VerifyResult{ + NodeName: aws.StringValue(instances.Reservations[0].Instances[0].PrivateDnsName), + }, nil +} diff --git a/upup/pkg/fi/cloudup/bootstrapchannelbuilder.go b/upup/pkg/fi/cloudup/bootstrapchannelbuilder.go index 81356e5763e86..2f3f27df250f6 100644 --- a/upup/pkg/fi/cloudup/bootstrapchannelbuilder.go +++ b/upup/pkg/fi/cloudup/bootstrapchannelbuilder.go @@ -304,7 +304,7 @@ func (b *BootstrapChannelBuilder) buildAddons() *channelsapi.Addons { // this manifest. For clusters whom are upgrading from RBAC to Node,RBAC the clusterrolebinding // will remain and have to be deleted manually once all the nodes have been upgraded. enableRBACAddon := true - if b.Cluster.Spec.NodeAuthorization != nil { + if b.UseKopsControllerForNodeBootstrap() || b.Cluster.Spec.NodeAuthorization != nil { enableRBACAddon = false } if b.Cluster.Spec.KubeAPIServer != nil { diff --git a/upup/pkg/fi/cloudup/template_functions.go b/upup/pkg/fi/cloudup/template_functions.go index 2b9ee91485fc0..bd5a6e68cc9c2 100644 --- a/upup/pkg/fi/cloudup/template_functions.go +++ b/upup/pkg/fi/cloudup/template_functions.go @@ -31,12 +31,14 @@ import ( "encoding/json" "fmt" "os" + "path" "strconv" "strings" "text/template" "github.com/Masterminds/sprig" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/sets" "k8s.io/klog" kopscontrollerconfig "k8s.io/kops/cmd/kops-controller/pkg/config" "k8s.io/kops/pkg/apis/kops" @@ -47,6 +49,7 @@ import ( "k8s.io/kops/pkg/resources/spotinst" "k8s.io/kops/pkg/wellknownports" "k8s.io/kops/upup/pkg/fi" + "k8s.io/kops/upup/pkg/fi/cloudup/awsup" "k8s.io/kops/upup/pkg/fi/cloudup/gce" "k8s.io/kops/util/pkg/env" ) @@ -379,6 +382,37 @@ func (tf *TemplateFunctions) KopsControllerConfig() (string, error) { ConfigBase: cluster.Spec.ConfigBase, } + if tf.UseKopsControllerForNodeBootstrap() { + pkiDir := "/etc/kubernetes/kops-controller/pki" + config.Server = &kopscontrollerconfig.ServerOptions{ + Listen: fmt.Sprintf(":%d", wellknownports.KopsControllerPort), + ServerCertificatePath: path.Join(pkiDir, "kops-controller.crt"), + ServerKeyPath: path.Join(pkiDir, "kops-controller.key"), + CABasePath: pkiDir, + SigningCAs: []string{fi.CertificateIDCA}, + } + + switch kops.CloudProviderID(cluster.Spec.CloudProvider) { + case kops.CloudProviderAWS: + nodesRoles := sets.String{} + for _, ig := range tf.InstanceGroups { + if ig.Spec.Role == kops.InstanceGroupRoleNode { + profile, err := tf.LinkToIAMInstanceProfile(ig) + if err != nil { + return "", fmt.Errorf("getting role for ig %s: %v", ig.Name, err) + } + nodesRoles.Insert(*profile.Name) + } + } + config.Server.Provider.AWS = &awsup.AWSVerifierOptions{ + NodesRoles: nodesRoles.List(), + Region: tf.Region, + } + default: + return "", fmt.Errorf("unsupported cloud provider %s", cluster.Spec.CloudProvider) + } + } + // To avoid indentation problems, we marshal as json. json is a subset of yaml b, err := json.Marshal(config) if err != nil { @@ -397,7 +431,7 @@ func (tf *TemplateFunctions) KopsControllerArgv() ([]string, error) { // Verbose, but not excessive logging argv = append(argv, "--v=2") - argv = append(argv, "--conf=/etc/kubernetes/kops-controller/config.yaml") + argv = append(argv, "--conf=/etc/kubernetes/kops-controller/config/config.yaml") return argv, nil } diff --git a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/amazonvpc/manifest.yaml b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/amazonvpc/manifest.yaml index aee0db3dccefa..cd68ede5d9cc9 100644 --- a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/amazonvpc/manifest.yaml +++ b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/amazonvpc/manifest.yaml @@ -7,7 +7,7 @@ spec: - id: k8s-1.16 kubernetesVersion: '>=1.16.0-alpha.0' manifest: kops-controller.addons.k8s.io/k8s-1.16.yaml - manifestHash: c0729abd228804c1083ed62a018e80c2de9c796c + manifestHash: 7a7039ba3b0e9c0027e486902fba3c6d266cbb46 name: kops-controller.addons.k8s.io selector: k8s-addon: kops-controller.addons.k8s.io diff --git a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/awsiamauthenticator/manifest.yaml b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/awsiamauthenticator/manifest.yaml index c8a2a192539d8..4e3f8b9c8d57c 100644 --- a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/awsiamauthenticator/manifest.yaml +++ b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/awsiamauthenticator/manifest.yaml @@ -7,7 +7,7 @@ spec: - id: k8s-1.16 kubernetesVersion: '>=1.16.0-alpha.0' manifest: kops-controller.addons.k8s.io/k8s-1.16.yaml - manifestHash: c0729abd228804c1083ed62a018e80c2de9c796c + manifestHash: 7a7039ba3b0e9c0027e486902fba3c6d266cbb46 name: kops-controller.addons.k8s.io selector: k8s-addon: kops-controller.addons.k8s.io diff --git a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/cilium/kops-controller.addons.k8s.io-k8s-1.16.yaml b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/cilium/kops-controller.addons.k8s.io-k8s-1.16.yaml index cacba3e380dd7..cd2920e40e0aa 100644 --- a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/cilium/kops-controller.addons.k8s.io-k8s-1.16.yaml +++ b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/cilium/kops-controller.addons.k8s.io-k8s-1.16.yaml @@ -35,7 +35,7 @@ spec: - command: - /kops-controller - --v=2 - - --conf=/etc/kubernetes/kops-controller/config.yaml + - --conf=/etc/kubernetes/kops-controller/config/config.yaml image: kope/kops-controller:1.19.0-alpha.2 name: kops-controller resources: @@ -45,8 +45,10 @@ spec: securityContext: runAsNonRoot: true volumeMounts: - - mountPath: /etc/kubernetes/kops-controller/ + - mountPath: /etc/kubernetes/kops-controller/config/ name: kops-controller-config + - mountPath: /etc/kubernetes/kops-controller/pki/ + name: kops-controller-pki dnsPolicy: Default hostNetwork: true nodeSelector: @@ -60,6 +62,10 @@ spec: - configMap: name: kops-controller name: kops-controller-config + - hostPath: + path: /etc/kubernetes/kops-controller/ + type: Directory + name: kops-controller-pki updateStrategy: rollingUpdate: maxUnavailable: 1 diff --git a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/cilium/manifest.yaml b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/cilium/manifest.yaml index e2b912ff5b84b..992b2e201c2c4 100644 --- a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/cilium/manifest.yaml +++ b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/cilium/manifest.yaml @@ -7,7 +7,7 @@ spec: - id: k8s-1.16 kubernetesVersion: '>=1.16.0-alpha.0' manifest: kops-controller.addons.k8s.io/k8s-1.16.yaml - manifestHash: c0729abd228804c1083ed62a018e80c2de9c796c + manifestHash: 7a7039ba3b0e9c0027e486902fba3c6d266cbb46 name: kops-controller.addons.k8s.io selector: k8s-addon: kops-controller.addons.k8s.io diff --git a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/simple/kops-controller.addons.k8s.io-k8s-1.16.yaml b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/simple/kops-controller.addons.k8s.io-k8s-1.16.yaml index cacba3e380dd7..cd2920e40e0aa 100644 --- a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/simple/kops-controller.addons.k8s.io-k8s-1.16.yaml +++ b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/simple/kops-controller.addons.k8s.io-k8s-1.16.yaml @@ -35,7 +35,7 @@ spec: - command: - /kops-controller - --v=2 - - --conf=/etc/kubernetes/kops-controller/config.yaml + - --conf=/etc/kubernetes/kops-controller/config/config.yaml image: kope/kops-controller:1.19.0-alpha.2 name: kops-controller resources: @@ -45,8 +45,10 @@ spec: securityContext: runAsNonRoot: true volumeMounts: - - mountPath: /etc/kubernetes/kops-controller/ + - mountPath: /etc/kubernetes/kops-controller/config/ name: kops-controller-config + - mountPath: /etc/kubernetes/kops-controller/pki/ + name: kops-controller-pki dnsPolicy: Default hostNetwork: true nodeSelector: @@ -60,6 +62,10 @@ spec: - configMap: name: kops-controller name: kops-controller-config + - hostPath: + path: /etc/kubernetes/kops-controller/ + type: Directory + name: kops-controller-pki updateStrategy: rollingUpdate: maxUnavailable: 1 diff --git a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/simple/manifest.yaml b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/simple/manifest.yaml index d9bf2dd63e0cd..7449805e34f1e 100644 --- a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/simple/manifest.yaml +++ b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/simple/manifest.yaml @@ -7,7 +7,7 @@ spec: - id: k8s-1.16 kubernetesVersion: '>=1.16.0-alpha.0' manifest: kops-controller.addons.k8s.io/k8s-1.16.yaml - manifestHash: c0729abd228804c1083ed62a018e80c2de9c796c + manifestHash: 7a7039ba3b0e9c0027e486902fba3c6d266cbb46 name: kops-controller.addons.k8s.io selector: k8s-addon: kops-controller.addons.k8s.io diff --git a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/weave/manifest.yaml b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/weave/manifest.yaml index 8d82d23d290e8..e203dffaa5f13 100644 --- a/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/weave/manifest.yaml +++ b/upup/pkg/fi/cloudup/tests/bootstrapchannelbuilder/weave/manifest.yaml @@ -7,7 +7,7 @@ spec: - id: k8s-1.16 kubernetesVersion: '>=1.16.0-alpha.0' manifest: kops-controller.addons.k8s.io/k8s-1.16.yaml - manifestHash: c0729abd228804c1083ed62a018e80c2de9c796c + manifestHash: 7a7039ba3b0e9c0027e486902fba3c6d266cbb46 name: kops-controller.addons.k8s.io selector: k8s-addon: kops-controller.addons.k8s.io diff --git a/upup/pkg/fi/nodeup/command.go b/upup/pkg/fi/nodeup/command.go index 43df77a21d2dc..9db3c700bd9a2 100644 --- a/upup/pkg/fi/nodeup/command.go +++ b/upup/pkg/fi/nodeup/command.go @@ -244,6 +244,7 @@ func (c *NodeUpCommand) Run(out io.Writer) error { loader.Builders = append(loader.Builders, &model.KubeSchedulerBuilder{NodeupModelContext: modelContext}) loader.Builders = append(loader.Builders, &model.EtcdManagerTLSBuilder{NodeupModelContext: modelContext}) loader.Builders = append(loader.Builders, &model.KubeProxyBuilder{NodeupModelContext: modelContext}) + loader.Builders = append(loader.Builders, &model.KopsControllerBuilder{NodeupModelContext: modelContext}) loader.Builders = append(loader.Builders, &networking.CommonBuilder{NodeupModelContext: modelContext}) loader.Builders = append(loader.Builders, &networking.CalicoBuilder{NodeupModelContext: modelContext}) @@ -251,6 +252,7 @@ func (c *NodeUpCommand) Run(out io.Writer) error { loader.Builders = append(loader.Builders, &networking.KuberouterBuilder{NodeupModelContext: modelContext}) loader.Builders = append(loader.Builders, &networking.LyftVPCBuilder{NodeupModelContext: modelContext}) + loader.Builders = append(loader.Builders, &model.BootstrapClientBuilder{NodeupModelContext: modelContext}) taskMap, err := loader.Build() if err != nil { return fmt.Errorf("error building loader: %v", err) diff --git a/upup/pkg/fi/nodeup/nodetasks/BUILD.bazel b/upup/pkg/fi/nodeup/nodetasks/BUILD.bazel index 676e74f8cc7ea..9424293adf214 100644 --- a/upup/pkg/fi/nodeup/nodetasks/BUILD.bazel +++ b/upup/pkg/fi/nodeup/nodetasks/BUILD.bazel @@ -5,6 +5,7 @@ go_library( srcs = [ "archive.go", "bindmount.go", + "bootstrap_client.go", "chattr.go", "createsdir.go", "file.go", @@ -22,9 +23,11 @@ go_library( deps = [ "//nodeup/pkg/distros:go_default_library", "//pkg/apis/kops:go_default_library", + "//pkg/apis/nodeup:go_default_library", "//pkg/backoff:go_default_library", "//pkg/kubeconfig:go_default_library", "//pkg/pki:go_default_library", + "//pkg/wellknownports:go_default_library", "//upup/pkg/fi:go_default_library", "//upup/pkg/fi/nodeup/cloudinit:go_default_library", "//upup/pkg/fi/nodeup/local:go_default_library", diff --git a/upup/pkg/fi/nodeup/nodetasks/bootstrap_client.go b/upup/pkg/fi/nodeup/nodetasks/bootstrap_client.go new file mode 100644 index 0000000000000..6bf19d00337b6 --- /dev/null +++ b/upup/pkg/fi/nodeup/nodetasks/bootstrap_client.go @@ -0,0 +1,193 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package nodetasks + +import ( + "bufio" + "bytes" + "crypto/rsa" + "crypto/tls" + "crypto/x509" + "encoding/json" + "encoding/pem" + "fmt" + "io/ioutil" + "net" + "net/http" + "net/url" + "strconv" + + "k8s.io/kops/pkg/apis/nodeup" + "k8s.io/kops/pkg/pki" + "k8s.io/kops/pkg/wellknownports" + "k8s.io/kops/upup/pkg/fi" +) + +type BootstrapClient struct { + // Authenticator generates authentication credentials for requests. + Authenticator fi.Authenticator + // CA is the CA certificate for kops-controller. + CA []byte + // Certs are the requested certificates. + Certs map[string]*BootstrapCert + + client *http.Client + keys map[string]*pki.PrivateKey +} + +type BootstrapCert struct { + Cert *fi.TaskDependentResource + Key *fi.TaskDependentResource +} + +var _ fi.Task = &BootstrapClient{} +var _ fi.HasName = &BootstrapClient{} +var _ fi.HasDependencies = &BootstrapClient{} + +func (b *BootstrapClient) GetDependencies(tasks map[string]fi.Task) []fi.Task { + return nil +} + +func (b *BootstrapClient) GetName() *string { + name := "BootstrapClient" + return &name +} + +func (b *BootstrapClient) String() string { + return "BootstrapClient" +} + +func (b *BootstrapClient) Run(c *fi.Context) error { + req := nodeup.BootstrapRequest{ + APIVersion: nodeup.BootstrapAPIVersion, + Certs: map[string]string{}, + } + + if b.keys == nil { + b.keys = map[string]*pki.PrivateKey{} + } + + for name, certRequest := range b.Certs { + key, ok := b.keys[name] + if !ok { + var err error + key, err = pki.GeneratePrivateKey() + if err != nil { + return fmt.Errorf("generating private key: %v", err) + } + + certRequest.Key.Resource = &asBytesResource{key} + b.keys[name] = key + } + + pkData, err := x509.MarshalPKIXPublicKey(key.Key.(*rsa.PrivateKey).Public()) + if err != nil { + return fmt.Errorf("marshalling public key: %v", err) + } + // TODO perhaps send a CSR instead to prove we own the private key? + req.Certs[name] = string(pem.EncodeToMemory(&pem.Block{Type: "RSA PUBLIC KEY", Bytes: pkData})) + } + + resp, err := b.queryBootstrap(c, &req) + if err != nil { + return err + } + + for name, certRequest := range b.Certs { + cert, ok := resp.Certs[name] + if !ok { + return fmt.Errorf("kops-controller did not return a %q certificate", name) + } + certificate, err := pki.ParsePEMCertificate([]byte(cert)) + if err != nil { + return fmt.Errorf("parsing %q certificate: %v", name, err) + } + certRequest.Cert.Resource = asBytesResource{certificate} + } + + return nil +} + +func (b *BootstrapClient) queryBootstrap(c *fi.Context, req *nodeup.BootstrapRequest) (*nodeup.BootstrapResponse, error) { + if b.client == nil { + certPool := x509.NewCertPool() + certPool.AppendCertsFromPEM(b.CA) + + b.client = &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{ + RootCAs: certPool, + MinVersion: tls.VersionTLS12, + }, + }, + } + } + + reqBytes, err := json.Marshal(req) + if err != nil { + return nil, err + } + + bootstrapUrl := url.URL{ + Scheme: "https", + Host: net.JoinHostPort(c.Cluster.Spec.MasterInternalName, strconv.Itoa(wellknownports.KopsControllerPort)), + Path: "/bootstrap", + } + httpReq, err := http.NewRequest("POST", bootstrapUrl.String(), bytes.NewReader(reqBytes)) + if err != nil { + return nil, err + } + httpReq.Header.Set("Content-Type", "application/json") + + token, err := b.Authenticator.CreateToken(reqBytes) + if err != nil { + return nil, err + } + httpReq.Header.Set("Authorization", token) + + resp, err := b.client.Do(httpReq) + if err != nil { + return nil, err + } + if resp.Body != nil { + defer resp.Body.Close() + } + + if resp.StatusCode != http.StatusOK { + detail := "" + if resp.Body != nil { + scanner := bufio.NewScanner(resp.Body) + if scanner.Scan() { + detail = scanner.Text() + } + } + return nil, fmt.Errorf("bootstrap returned status code %d: %s", resp.StatusCode, detail) + } + + var bootstrapResp nodeup.BootstrapResponse + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + err = json.Unmarshal(body, &bootstrapResp) + if err != nil { + return nil, err + } + + return &bootstrapResp, nil +}