Merge pull request kubernetes#15755 from johngmyers/nodeup-next

Remove use of ClusterSpec in nodeup

cmd/kops-controller/pkg/server/node_config.go

@@ -21,7 +21,6 @@ import (
 	"fmt"
 
 	"k8s.io/klog/v2"
-	"k8s.io/kops/pkg/apis/kops/registry"
 	"k8s.io/kops/pkg/apis/nodeup"
 	"k8s.io/kops/pkg/bootstrap"
 )
@@ -39,18 +38,6 @@ func (s *Server) getNodeConfig(ctx context.Context, req *nodeup.BootstrapRequest
 	// Note: For now, we're assuming there is only a single cluster, and it is ours.
 	// We therefore use the configured base path
 
-	// Today we load the full cluster config from the state store (e.g. S3) every time
-	// TODO: we should generate it on the fly (to allow for cluster reconfiguration)
-	{
-		p := s.configBase.Join(registry.PathClusterCompleted)
-
-		b, err := p.ReadFile(ctx)
-		if err != nil {
-			return nil, fmt.Errorf("error loading cluster config %q: %w", p, err)
-		}
-		nodeConfig.ClusterFullConfig = string(b)
-	}
-
 	{
 		p := s.configBase.Join("igconfig", "node", instanceGroupName, "nodeupconfig.yaml")
 

pkg/apis/nodeup/bootstrap.go

@@ -54,9 +54,6 @@ type BootstrapResponse struct {
 
 // NodeConfig holds configuration needed to boot a node (without the kops state store)
 type NodeConfig struct {
-	// ClusterFullConfig holds the completed configuration for the cluster.
-	ClusterFullConfig string `json:"clusterFullConfig,omitempty"`
-
 	// NodeupConfig holds the nodeup.Config for the node's instance group.
 	NodeupConfig string `json:"nodeupConfig,omitempty"`
 

pkg/apis/nodeup/config.go

@@ -44,6 +44,9 @@ type Config struct {
 	// Packages specifies additional packages to be installed.
 	Packages []string `json:"packages,omitempty"`
 
+	// ConfigStore configures the stores that nodes use to get their configuration when they don't use kops-controller.
+	ConfigStore *kops.ConfigStoreSpec `json:"configStore,omitempty"`
+
 	// EtcdClusterNames are the names of the etcd clusters.
 	EtcdClusterNames []string `json:",omitempty"`
 	// EtcdManifests are the manifests for running etcd.
@@ -365,6 +368,13 @@ func NewConfig(cluster *kops.Cluster, instanceGroup *kops.InstanceGroup) (*Confi
 		}
 	}
 
+	if instanceGroup.HasAPIServer() || !model.UseKopsControllerForNodeConfig(cluster) {
+		config.ConfigStore = &kops.ConfigStoreSpec{
+			Keypairs: cluster.Spec.ConfigStore.Keypairs,
+			Secrets:  cluster.Spec.ConfigStore.Secrets,
+		}
+	}
+
 	if instanceGroup.HasAPIServer() || cluster.UsesLegacyGossip() {
 		config.Networking.EgressProxy = cluster.Spec.Networking.EgressProxy
 	}

pkg/model/bootstrapscript.go

@@ -27,16 +27,14 @@ import (
 	"strings"
 
 	"k8s.io/klog/v2"
-	"k8s.io/kops/upup/pkg/fi/cloudup/scaleway"
-	"k8s.io/kops/upup/pkg/fi/utils"
-	"sigs.k8s.io/yaml"
-
 	"k8s.io/kops/pkg/apis/kops"
 	"k8s.io/kops/pkg/apis/nodeup"
 	"k8s.io/kops/pkg/model/resources"
 	"k8s.io/kops/upup/pkg/fi"
 	"k8s.io/kops/upup/pkg/fi/cloudup/awsup"
+	"k8s.io/kops/upup/pkg/fi/cloudup/scaleway"
 	"k8s.io/kops/upup/pkg/fi/fitasks"
+	"k8s.io/kops/upup/pkg/fi/utils"
 	"k8s.io/kops/util/pkg/architectures"
 	"k8s.io/kops/util/pkg/mirrors"
 )
@@ -341,49 +339,6 @@ func (b *BootstrapScript) Run(c *fi.CloudupContext) error {
 		nodeupScript.ProxyEnv = func() (string, error) {
 			return b.createProxyEnv(c.T.Cluster.Spec.Networking.EgressProxy)
 		}
-
-		nodeupScript.ClusterSpec = func() (string, error) {
-			cs := c.T.Cluster.Spec
-
-			spec := make(map[string]interface{})
-			spec["cloudConfig"] = cs.CloudConfig
-			spec["kubelet"] = cs.Kubelet
-
-			if b.ig.IsControlPlane() {
-				spec["encryptionConfig"] = cs.EncryptionConfig
-				spec["etcdClusters"] = make(map[string]kops.EtcdClusterSpec)
-				spec["kubeAPIServer"] = cs.KubeAPIServer
-				spec["kubeControllerManager"] = cs.KubeControllerManager
-				spec["kubeScheduler"] = cs.KubeScheduler
-				spec["masterKubelet"] = cs.ControlPlaneKubelet
-
-				for _, etcdCluster := range cs.EtcdClusters {
-					c := kops.EtcdClusterSpec{
-						Image:         etcdCluster.Image,
-						Version:       etcdCluster.Version,
-						Manager:       etcdCluster.Manager,
-						CPURequest:    etcdCluster.CPURequest,
-						MemoryRequest: etcdCluster.MemoryRequest,
-					}
-					for _, etcdMember := range etcdCluster.Members {
-						if fi.ValueOf(etcdMember.InstanceGroup) == b.ig.Name && etcdMember.VolumeSize != nil {
-							m := kops.EtcdMemberSpec{
-								Name:       etcdMember.Name,
-								VolumeSize: etcdMember.VolumeSize,
-							}
-							c.Members = append(c.Members, m)
-						}
-					}
-					spec["etcdClusters"].(map[string]kops.EtcdClusterSpec)[etcdCluster.Name] = c
-				}
-			}
-
-			content, err := yaml.Marshal(spec)
-			if err != nil {
-				return "", fmt.Errorf("error converting cluster spec to yaml for inclusion within bootstrap script: %v", err)
-			}
-			return string(content), nil
-		}
 	}
 
 	nodeupScript.CompressUserData = fi.ValueOf(b.ig.Spec.CompressUserData)

pkg/model/resources/nodeup.go

@@ -160,14 +160,6 @@ function download-release() {
 echo "== nodeup node config starting =="
 ensure-install-dir
 
-{{ if CompressUserData -}}
-echo "{{ GzipBase64 ClusterSpec }}" | base64 -d | gzip -d > conf/cluster_spec.yaml
-{{- else -}}
-cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC'
-{{ ClusterSpec }}
-__EOF_CLUSTER_SPEC
-{{- end }}
-
 {{ if CompressUserData -}}
 echo "{{ GzipBase64 KubeEnv }}" | base64 -d | gzip -d > conf/kube_env.yaml
 {{- else -}}
@@ -189,7 +181,6 @@ type NodeUpScript struct {
 	CloudProvider        string
 	ProxyEnv             func() (string, error)
 	EnvironmentVariables func() (string, error)
-	ClusterSpec          func() (string, error)
 }
 
 func funcEmptyString() (string, error) {
@@ -203,9 +194,6 @@ func (b *NodeUpScript) Build() (fi.Resource, error) {
 	if b.EnvironmentVariables == nil {
 		b.EnvironmentVariables = funcEmptyString
 	}
-	if b.ClusterSpec == nil {
-		b.ClusterSpec = funcEmptyString
-	}
 
 	functions := template.FuncMap{
 		"NodeUpSourceAmd64": func() string {
@@ -260,7 +248,6 @@ func (b *NodeUpScript) Build() (fi.Resource, error) {
 
 		"ProxyEnv":             b.ProxyEnv,
 		"EnvironmentVariables": b.EnvironmentVariables,
-		"ClusterSpec":          b.ClusterSpec,
 	}
 
 	return newTemplateResource("nodeup", nodeUpTemplate, functions, nil)

pkg/model/tests/data/bootstrapscript_0.txt

@@ -138,33 +138,11 @@ function download-release() {
 echo "== nodeup node config starting =="
 ensure-install-dir
 
-cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC'
-cloudConfig: null
-encryptionConfig: null
-etcdClusters:
-  events:
-    image: gcr.io/etcd-development/etcd:v3.1.11
-    version: 3.1.11
-  main:
-    version: 3.1.11
-kubeAPIServer:
-  image: CoreOS
-kubeControllerManager:
-  cloudProvider: aws
-kubeScheduler:
-  image: SomeImage
-kubelet:
-  kubeconfigPath: /etc/kubernetes/config.txt
-masterKubelet:
-  kubeconfigPath: /etc/kubernetes/config.cfg
-
-__EOF_CLUSTER_SPEC
-
 cat > conf/kube_env.yaml << '__EOF_KUBE_ENV'
 CloudProvider: aws
 InstanceGroupName: testIG
 InstanceGroupRole: ControlPlane
-NodeupConfigHash: NgbCLc+TuTI42Lom2jxgBLh5AZ1tgVEAhDKGbHLRw0k=
+NodeupConfigHash: od7B+hssN4a8ZCNx8Jdg6OzS3Jt8kMr6JOj8HdREE5E=
 
 __EOF_KUBE_ENV
 

pkg/model/tests/data/bootstrapscript_1.txt

@@ -138,33 +138,11 @@ function download-release() {
 echo "== nodeup node config starting =="
 ensure-install-dir
 
-cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC'
-cloudConfig: null
-encryptionConfig: null
-etcdClusters:
-  events:
-    image: gcr.io/etcd-development/etcd:v3.1.11
-    version: 3.1.11
-  main:
-    version: 3.1.11
-kubeAPIServer:
-  image: CoreOS
-kubeControllerManager:
-  cloudProvider: aws
-kubeScheduler:
-  image: SomeImage
-kubelet:
-  kubeconfigPath: /etc/kubernetes/config.txt
-masterKubelet:
-  kubeconfigPath: /etc/kubernetes/config.cfg
-
-__EOF_CLUSTER_SPEC
-
 cat > conf/kube_env.yaml << '__EOF_KUBE_ENV'
 CloudProvider: aws
 InstanceGroupName: testIG
 InstanceGroupRole: ControlPlane
-NodeupConfigHash: V2b4y8d/JuUHm6va1cT7qHLBwy3BErf2Frg+YZ692L8=
+NodeupConfigHash: h6qKgtEtS6PGukVzhRtiCAGz6RryXvF6RDTRbDMCdlA=
 
 __EOF_KUBE_ENV
 

pkg/model/tests/data/bootstrapscript_2.txt

@@ -138,33 +138,11 @@ function download-release() {
 echo "== nodeup node config starting =="
 ensure-install-dir
 
-cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC'
-cloudConfig: null
-encryptionConfig: null
-etcdClusters:
-  events:
-    image: gcr.io/etcd-development/etcd:v3.1.11
-    version: 3.1.11
-  main:
-    version: 3.1.11
-kubeAPIServer:
-  image: CoreOS
-kubeControllerManager:
-  cloudProvider: aws
-kubeScheduler:
-  image: SomeImage
-kubelet:
-  kubeconfigPath: /etc/kubernetes/config.txt
-masterKubelet:
-  kubeconfigPath: /etc/kubernetes/config.cfg
-
-__EOF_CLUSTER_SPEC
-
 cat > conf/kube_env.yaml << '__EOF_KUBE_ENV'
 CloudProvider: aws
 InstanceGroupName: testIG
 InstanceGroupRole: ControlPlane
-NodeupConfigHash: V2b4y8d/JuUHm6va1cT7qHLBwy3BErf2Frg+YZ692L8=
+NodeupConfigHash: h6qKgtEtS6PGukVzhRtiCAGz6RryXvF6RDTRbDMCdlA=
 
 __EOF_KUBE_ENV
 

pkg/model/tests/data/bootstrapscript_3.txt

@@ -138,13 +138,6 @@ function download-release() {
 echo "== nodeup node config starting =="
 ensure-install-dir
 
-cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC'
-cloudConfig: null
-kubelet:
-  kubeconfigPath: /etc/kubernetes/config.txt
-
-__EOF_CLUSTER_SPEC
-
 cat > conf/kube_env.yaml << '__EOF_KUBE_ENV'
 CloudProvider: aws
 InstanceGroupName: testIG

pkg/model/tests/data/bootstrapscript_4.txt

@@ -138,13 +138,6 @@ function download-release() {
 echo "== nodeup node config starting =="
 ensure-install-dir
 
-cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC'
-cloudConfig: null
-kubelet:
-  kubeconfigPath: /etc/kubernetes/config.txt
-
-__EOF_CLUSTER_SPEC
-
 cat > conf/kube_env.yaml << '__EOF_KUBE_ENV'
 CloudProvider: aws
 InstanceGroupName: testIG

pkg/model/tests/data/bootstrapscript_5.txt

@@ -138,13 +138,6 @@ function download-release() {
 echo "== nodeup node config starting =="
 ensure-install-dir
 
-cat > conf/cluster_spec.yaml << '__EOF_CLUSTER_SPEC'
-cloudConfig: null
-kubelet:
-  kubeconfigPath: /etc/kubernetes/config.txt
-
-__EOF_CLUSTER_SPEC
-
 cat > conf/kube_env.yaml << '__EOF_KUBE_ENV'
 CloudProvider: aws
 InstanceGroupName: testIG

pkg/model/tests/data/nodeupconfig_0.txt

@@ -39,6 +39,7 @@ Networking:
       port: 80
   nonMasqueradeCIDR: 10.100.0.0/16
 UpdatePolicy: automatic
+configStore: {}
 containerdConfig:
   logLevel: info
 docker:

pkg/model/tests/data/nodeupconfig_1.txt

@@ -57,6 +57,7 @@ Networking:
       port: 80
   nonMasqueradeCIDR: 10.100.0.0/16
 UpdatePolicy: automatic
+configStore: {}
 containerdConfig:
   logLevel: info
 docker:

pkg/model/tests/data/nodeupconfig_2.txt

@@ -57,6 +57,7 @@ Networking:
       port: 80
   nonMasqueradeCIDR: 10.100.0.0/16
 UpdatePolicy: automatic
+configStore: {}
 containerdConfig:
   logLevel: info
 docker: