Merge pull request #15125 from justinsb/node_challenge

Perform challenge callbacks into a node

cmd/kops-controller/pkg/server/server.go

@@ -36,6 +36,8 @@ import (
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/klog/v2"
 	"k8s.io/kops/cmd/kops-controller/pkg/config"
+	"k8s.io/kops/pkg/apis/kops"
+	"k8s.io/kops/pkg/apis/kops/model"
 	"k8s.io/kops/pkg/apis/nodeup"
 	"k8s.io/kops/pkg/bootstrap"
 	"k8s.io/kops/pkg/pki"
@@ -61,6 +63,9 @@ type Server struct {
 
 	// uncachedClient is an uncached client for the kube apiserver
 	uncachedClient client.Client
+
+	// challengeClient performs our callback-challenge into the node
+	challengeClient *bootstrap.ChallengeClient
 }
 
 var _ manager.LeaderElectionRunnable = &Server{}
@@ -94,6 +99,17 @@ func NewServer(opt *config.Options, verifier bootstrap.Verifier, uncachedClient
 	}
 	s.secretStore = secrets.NewVFSSecretStore(nil, p)
 
+	s.keystore, s.keypairIDs, err = newKeystore(opt.Server.CABasePath, opt.Server.SigningCAs)
+	if err != nil {
+		return nil, err
+	}
+
+	challengeClient, err := bootstrap.NewChallengeClient(s.keystore)
+	if err != nil {
+		return nil, err
+	}
+	s.challengeClient = challengeClient
+
 	r := http.NewServeMux()
 	r.Handle("/bootstrap", http.HandlerFunc(s.bootstrap))
 	server.Handler = recovery(r)
@@ -106,12 +122,6 @@ func (s *Server) NeedLeaderElection() bool {
 }
 
 func (s *Server) Start(ctx context.Context) error {
-	var err error
-	s.keystore, s.keypairIDs, err = newKeystore(s.opt.Server.CABasePath, s.opt.Server.SigningCAs)
-	if err != nil {
-		return err
-	}
-
 	go func() {
 		<-ctx.Done()
 
@@ -198,6 +208,17 @@ func (s *Server) bootstrap(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	if model.UseChallengeCallback(kops.CloudProviderID(s.opt.Cloud)) {
+		if err := s.challengeClient.DoCallbackChallenge(ctx, s.opt.ClusterName, id.ChallengeEndpoint, req); err != nil {
+			klog.Infof("bootstrap %s callback challenge failed: %v", r.RemoteAddr, err)
+			w.WriteHeader(http.StatusBadRequest)
+			_, _ = w.Write([]byte("callback failed"))
+			return
+		}
+
+		klog.Infof("performed successful callback challenge with %s; identified as %s", id.ChallengeEndpoint, id.NodeName)
+	}
+
 	resp := &nodeup.BootstrapResponse{
 		Certs: map[string]string{},
 	}

go.mod

@@ -54,6 +54,8 @@ require (
 	golang.org/x/sync v0.1.0
 	golang.org/x/sys v0.6.0
 	google.golang.org/api v0.112.0
+	google.golang.org/grpc v1.53.0
+	google.golang.org/protobuf v1.28.1
 	gopkg.in/gcfg.v1 v1.2.3
 	gopkg.in/inf.v0 v0.9.1
 	gopkg.in/square/go-jose.v2 v2.6.0
@@ -213,8 +215,6 @@ require (
 	gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/genproto v0.0.0-20230303212802-e74f57abe488 // indirect
-	google.golang.org/grpc v1.53.0 // indirect
-	google.golang.org/protobuf v1.28.1 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/warnings.v0 v0.1.2 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect

nodeup/pkg/model/bootstrap_client.go

@@ -49,7 +49,7 @@ func (b BootstrapClientBuilder) Build(c *fi.NodeupModelBuilderContext) error {
 	var authenticator bootstrap.Authenticator
 	var resolver resolver.Resolver
 
-	switch b.BootConfig.CloudProvider {
+	switch b.CloudProvider() {
 	case kops.CloudProviderAWS:
 		a, err := awsup.NewAWSAuthenticator(b.Cloud.Region())
 		if err != nil {
@@ -81,7 +81,7 @@ func (b BootstrapClientBuilder) Build(c *fi.NodeupModelBuilderContext) error {
 		authenticator = a
 
 	default:
-		return fmt.Errorf("unsupported cloud provider for authenticator %q", b.BootConfig.CloudProvider)
+		return fmt.Errorf("unsupported cloud provider for authenticator %q", b.CloudProvider())
 	}
 
 	baseURL := url.URL{
@@ -102,6 +102,8 @@ func (b BootstrapClientBuilder) Build(c *fi.NodeupModelBuilderContext) error {
 		Certs:      b.bootstrapCerts,
 		KeypairIDs: b.bootstrapKeypairIDs,
 	}
+	bootstrapClientTask.UseChallengeCallback = b.UseChallengeCallback(b.CloudProvider())
+	bootstrapClientTask.ClusterName = b.NodeupConfig.ClusterName
 
 	for _, cert := range b.bootstrapCerts {
 		cert.Cert.Task = bootstrapClientTask

nodeup/pkg/model/cloudconfig.go

@@ -90,7 +90,7 @@ func (b *CloudConfigBuilder) build(c *fi.NodeupModelBuilderContext, inTree bool)
 	// Add cloud config file if needed
 	var lines []string
 
-	cloudProvider := b.BootConfig.CloudProvider
+	cloudProvider := b.CloudProvider()
 
 	var config string
 	requireGlobal := true

nodeup/pkg/model/context.go

@@ -396,6 +396,11 @@ func (c *NodeupModelContext) UseKopsControllerForNodeBootstrap() bool {
 	return model.UseKopsControllerForNodeBootstrap(c.Cluster)
 }
 
+// UseChallengeCallback is true if we should use a callback challenge during node provisioning with kops-controller.
+func (c *NodeupModelContext) UseChallengeCallback(cloudProvider kops.CloudProviderID) bool {
+	return model.UseChallengeCallback(cloudProvider)
+}
+
 // UsesSecondaryIP checks if the CNI in use attaches secondary interfaces to the host.
 func (c *NodeupModelContext) UsesSecondaryIP() bool {
 	return (c.NodeupConfig.Networking.CNI != nil && c.NodeupConfig.Networking.CNI.UsesSecondaryIP) ||
@@ -630,14 +635,19 @@ func (c *NodeupModelContext) InstallNvidiaRuntime() bool {
 		c.GPUVendor == architectures.GPUVendorNvidia
 }
 
+// CloudProvider returns the cloud provider we are running on
+func (c *NodeupModelContext) CloudProvider() kops.CloudProviderID {
+	return c.BootConfig.CloudProvider
+}
+
 // RunningOnGCE returns true if we are running on GCE
 func (c *NodeupModelContext) RunningOnGCE() bool {
-	return c.BootConfig.CloudProvider == kops.CloudProviderGCE
+	return c.CloudProvider() == kops.CloudProviderGCE
 }
 
 // RunningOnAzure returns true if we are running on Azure
 func (c *NodeupModelContext) RunningOnAzure() bool {
-	return c.BootConfig.CloudProvider == kops.CloudProviderAzure
+	return c.CloudProvider() == kops.CloudProviderAzure
 }
 
 // GetMetadataLocalIP returns the local IP address read from metadata

nodeup/pkg/model/kube_apiserver.go

@@ -67,7 +67,7 @@ func (b *KubeAPIServerBuilder) Build(c *fi.NodeupModelBuilderContext) error {
 		kubeAPIServer = *b.NodeupConfig.APIServerConfig.KubeAPIServer
 	}
 
-	if b.BootConfig.CloudProvider == kops.CloudProviderHetzner {
+	if b.CloudProvider() == kops.CloudProviderHetzner {
 		localIP, err := b.GetMetadataLocalIP()
 		if err != nil {
 			return err
@@ -419,7 +419,7 @@ func (b *KubeAPIServerBuilder) writeServerCertificate(c *fi.NodeupModelBuilderCo
 		// We also want to be able to reference it locally via https://127.0.0.1
 		alternateNames = append(alternateNames, "127.0.0.1")
 
-		if b.BootConfig.CloudProvider == kops.CloudProviderHetzner {
+		if b.CloudProvider() == kops.CloudProviderHetzner {
 			localIP, err := b.GetMetadataLocalIP()
 			if err != nil {
 				return err
@@ -428,7 +428,7 @@ func (b *KubeAPIServerBuilder) writeServerCertificate(c *fi.NodeupModelBuilderCo
 				alternateNames = append(alternateNames, localIP)
 			}
 		}
-		if b.BootConfig.CloudProvider == kops.CloudProviderOpenstack {
+		if b.CloudProvider() == kops.CloudProviderOpenstack {
 			instanceAddress, err := getInstanceAddress()
 			if err != nil {
 				return err

nodeup/pkg/model/kubelet.go

@@ -737,7 +737,7 @@ func (b *KubeletBuilder) buildKubeletServingCertificate(c *fi.NodeupModelBuilder
 }
 
 func (b *KubeletBuilder) kubeletNames() ([]string, error) {
-	if b.BootConfig.CloudProvider != kops.CloudProviderAWS {
+	if b.CloudProvider() != kops.CloudProviderAWS {
 		name, err := os.Hostname()
 		if err != nil {
 			return nil, err

nodeup/pkg/model/ntp.go

@@ -50,7 +50,7 @@ func (b *NTPBuilder) Build(c *fi.NodeupModelBuilderContext) error {
 	}
 
 	var ntpHost string
-	switch b.BootConfig.CloudProvider {
+	switch b.CloudProvider() {
 	case kops.CloudProviderAWS:
 		ntpHost = "169.254.169.123"
 	case kops.CloudProviderGCE:

nodeup/pkg/model/prefix.go

@@ -34,15 +34,15 @@ func (b *PrefixBuilder) Build(c *fi.NodeupModelBuilderContext) error {
 	if !b.IsKopsControllerIPAM() {
 		return nil
 	}
-	switch b.BootConfig.CloudProvider {
+	switch b.CloudProvider() {
 	case kops.CloudProviderAWS:
 		c.AddTask(&nodetasks.Prefix{
 			Name: "prefix",
 		})
 	case kops.CloudProviderGCE:
 		// Prefix is assigned by GCE
 	default:
-		return fmt.Errorf("kOps IPAM controller not supported on cloud %q", b.BootConfig.CloudProvider)
+		return fmt.Errorf("kOps IPAM controller not supported on cloud %q", b.CloudProvider())
 	}
 	return nil
 }

nodeup/pkg/model/protokube.go

@@ -176,7 +176,7 @@ type ProtokubeFlags struct {
 func (t *ProtokubeBuilder) ProtokubeFlags() (*ProtokubeFlags, error) {
 	f := &ProtokubeFlags{
 		Channels:      t.NodeupConfig.Channels,
-		Cloud:         fi.PtrTo(string(t.BootConfig.CloudProvider)),
+		Cloud:         fi.PtrTo(string(t.CloudProvider())),
 		Containerized: fi.PtrTo(false),
 		LogLevel:      fi.PtrTo(int32(4)),
 		Master:        b(t.IsMaster),
@@ -273,7 +273,7 @@ func (t *ProtokubeBuilder) buildEnvFile() (*nodetasks.File, error) {
 		}
 	}
 
-	if t.BootConfig.CloudProvider == kops.CloudProviderDO && os.Getenv("DIGITALOCEAN_ACCESS_TOKEN") != "" {
+	if t.CloudProvider() == kops.CloudProviderDO && os.Getenv("DIGITALOCEAN_ACCESS_TOKEN") != "" {
 		envVars["DIGITALOCEAN_ACCESS_TOKEN"] = os.Getenv("DIGITALOCEAN_ACCESS_TOKEN")
 	}
 
@@ -294,7 +294,7 @@ func (t *ProtokubeBuilder) buildEnvFile() (*nodetasks.File, error) {
 		envVars["AZURE_STORAGE_ACCOUNT"] = os.Getenv("AZURE_STORAGE_ACCOUNT")
 	}
 
-	if t.BootConfig.CloudProvider == kops.CloudProviderScaleway {
+	if t.CloudProvider() == kops.CloudProviderScaleway {
 		if os.Getenv("SCW_PROFILE") != "" || os.Getenv("SCW_SECRET_KEY") != "" {
 			profile, err := scaleway.CreateValidScalewayProfile()
 			if err != nil {

nodeup/pkg/model/sysctls.go

@@ -125,7 +125,7 @@ func (b *SysctlBuilder) Build(c *fi.NodeupModelBuilderContext) error {
 		)
 	}
 
-	if b.BootConfig.CloudProvider == kops.CloudProviderAWS {
+	if b.CloudProvider() == kops.CloudProviderAWS {
 		sysctls = append(sysctls,
 			"# AWS settings",
 			"",

nodeup/pkg/model/warm_pool.go

@@ -30,7 +30,7 @@ var _ fi.NodeupModelBuilder = &WarmPoolBuilder{}
 
 func (b *WarmPoolBuilder) Build(c *fi.NodeupModelBuilderContext) error {
 	// Check if the cloud provider is AWS
-	if b.BootConfig.CloudProvider != kops.CloudProviderAWS {
+	if b.CloudProvider() != kops.CloudProviderAWS {
 		return nil
 	}
 

pkg/apis/kops/model/features.go

@@ -36,6 +36,16 @@ func UseKopsControllerForNodeBootstrap(cluster *kops.Cluster) bool {
 	}
 }
 
+// UseChallengeCallback is true if we should use a callback challenge during node provisioning with kops-controller.
+func UseChallengeCallback(cloudProvider kops.CloudProviderID) bool {
+	switch cloudProvider {
+	case kops.CloudProviderHetzner:
+		return true
+	default:
+		return false
+	}
+}
+
 // UseKopsControllerForNodeConfig checks if nodeup should use kops-controller to get nodeup.Config.
 func UseKopsControllerForNodeConfig(cluster *kops.Cluster) bool {
 	switch cluster.Spec.GetCloudProvider() {

pkg/apis/nodeup/bootstrap.go

@@ -30,6 +30,17 @@ type BootstrapRequest struct {
 	// IncludeNodeConfig controls whether the cluster & instance group configuration should be returned.
 	// This allows for nodes without access to the kops state store.
 	IncludeNodeConfig bool `json:"includeNodeConfig"`
+
+	// Challenge is for a callback challenge.
+	Challenge *ChallengeRequest `json:"challenge,omitempty"`
+}
+
+// ChallengeRequest describes the callback challenge.
+type ChallengeRequest struct {
+	Endpoint        string `json:"endpoint,omitempty"`
+	ServerCA        []byte `json:"ca,omitempty"`
+	ChallengeID     string `json:"challengeID,omitempty"`
+	ChallengeSecret []byte `json:"challengeSecret,omitempty"`
 }
 
 // BootstrapResponse is a response to a BootstrapRequest.

pkg/apis/nodeup/config.go

@@ -127,6 +127,8 @@ type BootConfig struct {
 	// APIServerIPs is the API server IP addresses.
 	// This field is used for adding an alias for api.internal. in /etc/hosts, when Topology.DNS.Type == DNSTypeNone.
 	APIServerIPs []string `json:",omitempty"`
+	// ClusterName is the name of the cluster.
+	ClusterName string `json:",omitempty"`
 	// InstanceGroupName is the name of the instance group.
 	InstanceGroupName string `json:",omitempty"`
 	// InstanceGroupRole is the instance group role.
@@ -200,6 +202,7 @@ func NewConfig(cluster *kops.Cluster, instanceGroup *kops.InstanceGroup) (*Confi
 
 	bootConfig := BootConfig{
 		CloudProvider:     cluster.Spec.GetCloudProvider(),
+		ClusterName:       cluster.ObjectMeta.Name,
 		InstanceGroupName: instanceGroup.ObjectMeta.Name,
 		InstanceGroupRole: role,
 	}

pkg/bootstrap/authenticate.go

@@ -39,6 +39,12 @@ type VerifyResult struct {
 
 	// CertificateNames is the alternate names the node is authorized to use for certificates.
 	CertificateNames []string
+
+	// ChallengeEndpoint is a valid endpoints to which we should issue a challenge request,
+	// corresponding to the node the request identified as.
+	// This should be sourced from e.g. the cloud, and acts as a cross-check
+	// that this is the correct instance.
+	ChallengeEndpoint string
 }
 
 // Verifier verifies authentication credentials for requests.