diff --git a/cmd/kops-controller/BUILD.bazel b/cmd/kops-controller/BUILD.bazel index 7d6196e352cc6..6aa8596fde76c 100644 --- a/cmd/kops-controller/BUILD.bazel +++ b/cmd/kops-controller/BUILD.bazel @@ -9,11 +9,14 @@ go_library( "//cmd/kops-controller/controllers:go_default_library", "//cmd/kops-controller/pkg/config:go_default_library", "//cmd/kops-controller/pkg/server:go_default_library", + "//pkg/apis/kops:go_default_library", "//pkg/nodeidentity:go_default_library", "//pkg/nodeidentity/aws:go_default_library", "//pkg/nodeidentity/do:go_default_library", "//pkg/nodeidentity/gce:go_default_library", "//pkg/nodeidentity/openstack:go_default_library", + "//upup/pkg/fi:go_default_library", + "//upup/pkg/fi/cloudup/awsup:go_default_library", "//vendor/k8s.io/api/core/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library", "//vendor/k8s.io/client-go/plugin/pkg/client/auth/gcp:go_default_library", diff --git a/cmd/kops-controller/main.go b/cmd/kops-controller/main.go index d2e8f55760077..6a5e4b4e1c67f 100644 --- a/cmd/kops-controller/main.go +++ b/cmd/kops-controller/main.go @@ -30,11 +30,14 @@ import ( "k8s.io/kops/cmd/kops-controller/controllers" "k8s.io/kops/cmd/kops-controller/pkg/config" "k8s.io/kops/cmd/kops-controller/pkg/server" + "k8s.io/kops/pkg/apis/kops" "k8s.io/kops/pkg/nodeidentity" nodeidentityaws "k8s.io/kops/pkg/nodeidentity/aws" nodeidentitydo "k8s.io/kops/pkg/nodeidentity/do" nodeidentitygce "k8s.io/kops/pkg/nodeidentity/gce" nodeidentityos "k8s.io/kops/pkg/nodeidentity/openstack" + "k8s.io/kops/upup/pkg/fi" + "k8s.io/kops/upup/pkg/fi/cloudup/awsup" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/yaml" @@ -83,7 +86,18 @@ func main() { ctrl.SetLogger(klogr.New()) if opt.Server != nil { - srv, err := server.NewServer(&opt) + var verifier fi.Verifier + var err error + switch opt.Server.Provider { + case kops.CloudProviderAWS: + verifier, err = awsup.NewAWSVerifier() + if err != nil { + setupLog.Error(err, "unable to create verifier") + os.Exit(1) + } + } + + srv, err := server.NewServer(&opt, verifier) if err != nil { setupLog.Error(err, "unable to create server") os.Exit(1) diff --git a/cmd/kops-controller/pkg/config/BUILD.bazel b/cmd/kops-controller/pkg/config/BUILD.bazel index 093d96826049e..dd35d7f59abb8 100644 --- a/cmd/kops-controller/pkg/config/BUILD.bazel +++ b/cmd/kops-controller/pkg/config/BUILD.bazel @@ -5,4 +5,5 @@ go_library( srcs = ["options.go"], importpath = "k8s.io/kops/cmd/kops-controller/pkg/config", visibility = ["//visibility:public"], + deps = ["//pkg/apis/kops:go_default_library"], ) diff --git a/cmd/kops-controller/pkg/config/options.go b/cmd/kops-controller/pkg/config/options.go index fa98e935f3175..b2ed9138b3439 100644 --- a/cmd/kops-controller/pkg/config/options.go +++ b/cmd/kops-controller/pkg/config/options.go @@ -16,6 +16,8 @@ limitations under the License. package config +import "k8s.io/kops/pkg/apis/kops" + type Options struct { Cloud string `json:"cloud,omitempty"` ConfigBase string `json:"configBase,omitempty"` @@ -29,6 +31,9 @@ type ServerOptions struct { // Listen is the network endpoint (ip and port) we should listen on. Listen string + // Provider is the cloud provider. + Provider kops.CloudProviderID `json:"provider"` + // ServerKeyPath is the path to our TLS serving private key. ServerKeyPath string `json:"serverKeyPath,omitempty"` // ServerCertificatePath is the path to our TLS serving certificate. diff --git a/cmd/kops-controller/pkg/server/BUILD.bazel b/cmd/kops-controller/pkg/server/BUILD.bazel index 9c65dc3f40cc0..8ba89610856bb 100644 --- a/cmd/kops-controller/pkg/server/BUILD.bazel +++ b/cmd/kops-controller/pkg/server/BUILD.bazel @@ -8,6 +8,7 @@ go_library( deps = [ "//cmd/kops-controller/pkg/config:go_default_library", "//pkg/apis/nodeup:go_default_library", + "//upup/pkg/fi:go_default_library", "//vendor/github.com/gorilla/mux:go_default_library", "//vendor/k8s.io/klog:go_default_library", ], diff --git a/cmd/kops-controller/pkg/server/server.go b/cmd/kops-controller/pkg/server/server.go index ad24a90f9a38c..223ac002591b1 100644 --- a/cmd/kops-controller/pkg/server/server.go +++ b/cmd/kops-controller/pkg/server/server.go @@ -20,6 +20,7 @@ import ( "crypto/tls" "encoding/json" "fmt" + "io/ioutil" "net/http" "runtime/debug" @@ -27,14 +28,16 @@ import ( "k8s.io/klog" "k8s.io/kops/cmd/kops-controller/pkg/config" "k8s.io/kops/pkg/apis/nodeup" + "k8s.io/kops/upup/pkg/fi" ) type Server struct { - opt *config.Options - server *http.Server + opt *config.Options + server *http.Server + verifier fi.Verifier } -func NewServer(opt *config.Options) (*Server, error) { +func NewServer(opt *config.Options, verifier fi.Verifier) (*Server, error) { server := &http.Server{ Addr: opt.Server.Listen, TLSConfig: &tls.Config{ @@ -44,8 +47,9 @@ func NewServer(opt *config.Options) (*Server, error) { } s := &Server{ - opt: opt, - server: server, + opt: opt, + server: server, + verifier: verifier, } r := mux.NewRouter() r.Handle("/bootstrap", http.HandlerFunc(s.bootstrap)) @@ -65,10 +69,26 @@ func (s *Server) bootstrap(w http.ResponseWriter, r *http.Request) { return } - // TODO: authenticate request + body, err := ioutil.ReadAll(r.Body) + if err != nil { + klog.Infof("bootstrap %s read err: %v", r.RemoteAddr, err) + w.WriteHeader(http.StatusBadRequest) + _, _ = w.Write([]byte(fmt.Sprintf("bootstrap %s failed to read body: %v", r.RemoteAddr, err))) + return + } + + id, err := s.verifier.VerifyToken(r.Header.Get("Authorization"), body) + if err != nil { + klog.Infof("bootstrap %s verify err: %v", r.RemoteAddr, err) + w.WriteHeader(http.StatusForbidden) + _, _ = w.Write([]byte(fmt.Sprintf("failed to verify token: %v", err))) + return + } + + klog.Infof("id is %s", id) // todo do something with id req := &nodeup.BootstrapRequest{} - err := json.NewDecoder(r.Body).Decode(req) + err = json.Unmarshal(body, req) if err != nil { klog.Infof("bootstrap %s decode err: %v", r.RemoteAddr, err) w.WriteHeader(http.StatusBadRequest) diff --git a/nodeup/pkg/model/bootstrap_client.go b/nodeup/pkg/model/bootstrap_client.go index 699a901d54467..be46b9ced233a 100644 --- a/nodeup/pkg/model/bootstrap_client.go +++ b/nodeup/pkg/model/bootstrap_client.go @@ -17,7 +17,9 @@ limitations under the License. package model import ( + "k8s.io/kops/pkg/apis/kops" "k8s.io/kops/upup/pkg/fi" + "k8s.io/kops/upup/pkg/fi/cloudup/awsup" "k8s.io/kops/upup/pkg/fi/nodeup/nodetasks" ) @@ -31,13 +33,24 @@ func (b BootstrapClientBuilder) Build(c *fi.ModelBuilderContext) error { return nil } + var authenticator fi.Authenticator + var err error + switch kops.CloudProviderID(b.Cluster.Spec.CloudProvider) { + case kops.CloudProviderAWS: + authenticator, err = awsup.NewAWSAuthenticator() + } + if err != nil { + return err + } + cert, err := b.GetCert(fi.CertificateIDCA) if err != nil { return err } bootstrapClient := &nodetasks.BootstrapClient{ - CA: cert, + Authenticator: authenticator, + CA: cert, } c.AddTask(bootstrapClient) return nil diff --git a/upup/pkg/fi/BUILD.bazel b/upup/pkg/fi/BUILD.bazel index 021c74f44ae7e..f8a7e2db0b4ce 100644 --- a/upup/pkg/fi/BUILD.bazel +++ b/upup/pkg/fi/BUILD.bazel @@ -4,6 +4,7 @@ go_library( name = "go_default_library", srcs = [ "assetstore.go", + "authenticate.go", "ca.go", "changes.go", "clientset_castore.go", diff --git a/upup/pkg/fi/authenticate.go b/upup/pkg/fi/authenticate.go new file mode 100644 index 0000000000000..874beb592f6ca --- /dev/null +++ b/upup/pkg/fi/authenticate.go @@ -0,0 +1,27 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package fi + +// Authenticator generates authentication credentials for requests. +type Authenticator interface { + CreateToken(body []byte) (string, error) +} + +// Verifier verifies authentication credentials for requests. +type Verifier interface { + VerifyToken(token string, body []byte) (string, error) +} diff --git a/upup/pkg/fi/cloudup/awsup/BUILD.bazel b/upup/pkg/fi/cloudup/awsup/BUILD.bazel index c5e5d5a811b6c..4d3dca315b77b 100644 --- a/upup/pkg/fi/cloudup/awsup/BUILD.bazel +++ b/upup/pkg/fi/cloudup/awsup/BUILD.bazel @@ -4,8 +4,10 @@ go_library( name = "go_default_library", srcs = [ "aws_apitarget.go", + "aws_authenticator.go", "aws_cloud.go", "aws_utils.go", + "aws_verifier.go", "instancegroups.go", "logging_retryer.go", "machine_types.go", @@ -44,6 +46,7 @@ go_library( "//vendor/github.com/aws/aws-sdk-go/service/iam/iamiface:go_default_library", "//vendor/github.com/aws/aws-sdk-go/service/route53:go_default_library", "//vendor/github.com/aws/aws-sdk-go/service/route53/route53iface:go_default_library", + "//vendor/github.com/aws/aws-sdk-go/service/sts:go_default_library", "//vendor/k8s.io/api/core/v1:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//vendor/k8s.io/klog:go_default_library", diff --git a/upup/pkg/fi/cloudup/awsup/aws_authenticator.go b/upup/pkg/fi/cloudup/awsup/aws_authenticator.go new file mode 100644 index 0000000000000..7f3e1bc7cf5d5 --- /dev/null +++ b/upup/pkg/fi/cloudup/awsup/aws_authenticator.go @@ -0,0 +1,62 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package awsup + +import ( + "crypto/sha256" + "encoding/base64" + "encoding/json" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/sts" + "k8s.io/kops/upup/pkg/fi" +) + +const AWSAuthenticationTokenPrefix = "x-aws-sts " + +type awsAuthenticator struct { + sts *sts.STS +} + +var _ fi.Authenticator = &awsAuthenticator{} + +func NewAWSAuthenticator() (fi.Authenticator, error) { + config := aws.NewConfig().WithCredentialsChainVerboseErrors(true) + sess, err := session.NewSession(config) + if err != nil { + return nil, err + } + return &awsAuthenticator{ + sts: sts.New(sess), + }, nil +} + +func (a awsAuthenticator) CreateToken(body []byte) (string, error) { + sha := sha256.Sum256(body) + + stsRequest, _ := a.sts.GetCallerIdentityRequest(nil) + + stsRequest.HTTPRequest.Header.Add("X-Kops-Request-SHA", base64.RawStdEncoding.EncodeToString(sha[:])) + err := stsRequest.Sign() + if err != nil { + return "", err + } + + headers, _ := json.Marshal(stsRequest.HTTPRequest.Header) + return AWSAuthenticationTokenPrefix + base64.StdEncoding.EncodeToString(headers), nil +} diff --git a/upup/pkg/fi/cloudup/awsup/aws_verifier.go b/upup/pkg/fi/cloudup/awsup/aws_verifier.go new file mode 100644 index 0000000000000..1f7f0815d34ef --- /dev/null +++ b/upup/pkg/fi/cloudup/awsup/aws_verifier.go @@ -0,0 +1,149 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package awsup + +import ( + "bytes" + "crypto/sha256" + "encoding/base64" + "encoding/json" + "encoding/xml" + "fmt" + "io" + "io/ioutil" + "net" + "net/http" + "strconv" + "strings" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/sts" + "k8s.io/kops/upup/pkg/fi" +) + +type awsVerifier struct { + sts *sts.STS + client http.Client +} + +var _ fi.Verifier = &awsVerifier{} + +func NewAWSVerifier() (fi.Verifier, error) { + config := aws.NewConfig().WithCredentialsChainVerboseErrors(true) + sess, err := session.NewSession(config) + if err != nil { + return nil, err + } + return &awsVerifier{ + sts: sts.New(sess), + client: http.Client{ + Transport: &http.Transport{ + Proxy: http.ProxyFromEnvironment, + DialContext: (&net.Dialer{ + Timeout: 30 * time.Second, + }).DialContext, + ForceAttemptHTTP2: true, + DisableKeepAlives: true, + MaxIdleConnsPerHost: -1, + TLSHandshakeTimeout: 10 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + }, + CheckRedirect: func(req *http.Request, via []*http.Request) error { + return http.ErrUseLastResponse + }, + }, + }, nil +} + +type GetCallerIdentityResponse struct { + XMLName xml.Name `xml:"GetCallerIdentityResponse"` + GetCallerIdentityResult []GetCallerIdentityResult `xml:"GetCallerIdentityResult"` + ResponseMetadata []ResponseMetadata `xml:"ResponseMetadata"` +} + +type GetCallerIdentityResult struct { + Arn string `xml:"Arn"` + UserId string `xml:"UserId"` + Account string `xml:"Account"` +} + +type ResponseMetadata struct { + RequestId string `xml:"RequestId"` +} + +func (a awsVerifier) VerifyToken(token string, body []byte) (string, error) { + if !strings.HasPrefix(token, AWSAuthenticationTokenPrefix) { + return "", fmt.Errorf("incorrect authorization type") + } + token = strings.TrimPrefix(token, AWSAuthenticationTokenPrefix) + + // We rely on the client and server using the same version of the same STS library. + stsRequest, _ := a.sts.GetCallerIdentityRequest(nil) + err := stsRequest.Sign() + if err != nil { + return "", fmt.Errorf("creating identity request: %v", err) + } + + stsRequest.HTTPRequest.Header = nil + tokenBytes, err := base64.StdEncoding.DecodeString(token) + if err != nil { + return "", fmt.Errorf("decoding authorization token: %v", err) + } + err = json.Unmarshal(tokenBytes, &stsRequest.HTTPRequest.Header) + if err != nil { + return "", fmt.Errorf("unmarshalling authorization token: %v", err) + } + + sha := sha256.Sum256(body) + if stsRequest.HTTPRequest.Header.Get("X-Kops-Request-SHA") != base64.RawStdEncoding.EncodeToString(sha[:]) { + return "", fmt.Errorf("incorrect SHA") + } + + requestBytes, _ := ioutil.ReadAll(stsRequest.Body) + _, _ = stsRequest.Body.Seek(0, io.SeekStart) + if stsRequest.HTTPRequest.Header.Get("Content-Length") != strconv.Itoa(len(requestBytes)) { + return "", fmt.Errorf("incorrect content-length") + } + + // TODO - implement retry? + response, err := a.client.Do(stsRequest.HTTPRequest) + if err != nil { + return "", fmt.Errorf("sending STS request: %v", err) + } + if response != nil { + defer response.Body.Close() + } + + responseBody, err := ioutil.ReadAll(response.Body) + if err != nil { + return "", fmt.Errorf("reading STS response: %v", err) + } + if response.StatusCode != 200 { + return "", fmt.Errorf("received status code %d from STS: %s", response.StatusCode, string(responseBody)) + } + + result := GetCallerIdentityResponse{} + err = xml.NewDecoder(bytes.NewReader(responseBody)).Decode(&result) + if err != nil { + return "", fmt.Errorf("decoding STS response: %v", err) + } + + marshal, _ := json.Marshal(result) + return string(marshal), nil +} diff --git a/upup/pkg/fi/cloudup/template_functions.go b/upup/pkg/fi/cloudup/template_functions.go index 8b7eb4315d1ce..662d9966e2bed 100644 --- a/upup/pkg/fi/cloudup/template_functions.go +++ b/upup/pkg/fi/cloudup/template_functions.go @@ -392,6 +392,7 @@ func (tf *TemplateFunctions) KopsControllerConfig() (string, error) { pkiDir := "/etc/kubernetes/kops-controller/pki" config.Server = &kopscontrollerconfig.ServerOptions{ Listen: fmt.Sprintf(":%d", wellknownports.KopsControllerPort), + Provider: kops.CloudProviderID(cluster.Spec.CloudProvider), ServerCertificatePath: path.Join(pkiDir, "kops-controller.crt"), ServerKeyPath: path.Join(pkiDir, "kops-controller.key"), } diff --git a/upup/pkg/fi/nodeup/nodetasks/bootstrap_client.go b/upup/pkg/fi/nodeup/nodetasks/bootstrap_client.go index fc5ca905b6e91..ff1ec0b5bd7ae 100644 --- a/upup/pkg/fi/nodeup/nodetasks/bootstrap_client.go +++ b/upup/pkg/fi/nodeup/nodetasks/bootstrap_client.go @@ -35,6 +35,8 @@ import ( ) type BootstrapClient struct { + // Authenticator generates authentication credentials for requests. + Authenticator fi.Authenticator // CA is the CA certificate for kops-controller. CA []byte @@ -43,6 +45,11 @@ type BootstrapClient struct { var _ fi.Task = &BootstrapClient{} var _ fi.HasName = &BootstrapClient{} +var _ fi.HasDependencies = &BootstrapClient{} + +func (b *BootstrapClient) GetDependencies(tasks map[string]fi.Task) []fi.Task { + return nil +} func (b *BootstrapClient) GetName() *string { name := "BootstrapClient" @@ -91,7 +98,19 @@ func (b *BootstrapClient) queryBootstrap(c *fi.Context, req nodeup.BootstrapRequ Host: net.JoinHostPort(c.Cluster.Spec.MasterInternalName, strconv.Itoa(wellknownports.KopsControllerPort)), Path: "/bootstrap", } - resp, err := b.client.Post(bootstrapUrl.String(), "application/json", bytes.NewReader(reqBytes)) + httpReq, err := http.NewRequest("POST", bootstrapUrl.String(), bytes.NewReader(reqBytes)) + if err != nil { + return err + } + httpReq.Header.Set("Content-Type", "application/json") + + token, err := b.Authenticator.CreateToken(reqBytes) + if err != nil { + return err + } + httpReq.Header.Set("Authorization", token) + + resp, err := b.client.Do(httpReq) if err != nil { return err }