Skip to content

Write full ClusterConfiguration to disk instead of just version string #3

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
AcidLeroy wants to merge 4 commits into
base: kubeadm-version
Choose a base branch
from
Open

Write full ClusterConfiguration to disk instead of just version string #3

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
c5baa39
Write kubeadm version file for worker nodes joining during CP upgrade
AcidLeroy Mar 10, 2026
b9466e1
First round of reviews addressed
AcidLeroy Mar 11, 2026
d8f6d0c
Write full ClusterConfiguration to disk instead of just version string
AcidLeroy Mar 11, 2026
75ac6e4
Register KubeadmControlPlane types in bootstrap controller scheme
AcidLeroy Mar 11, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions bootstrap/kubeadm/config/rbac/role.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -93,3 +93,12 @@ rules:
- get
- list
- watch
- apiGroups:
- controlplane.cluster.x-k8s.io
resources:
- kubeadmcontrolplanes
- kubeadmcontrolplanes/status
verbs:
- get
- list
- watch
5 changes: 4 additions & 1 deletion bootstrap/kubeadm/internal/cloudinit/cloudinit_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -305,7 +305,8 @@ func TestNewJoinNodeCommands(t *testing.T) {
Users: nil,
NTP: nil,
},
JoinConfiguration: "my-join-config",
JoinConfiguration: "my-join-config",
ClusterConfigurationYAML: "apiVersion: kubeadm.k8s.io/v1beta4\nkind: ClusterConfiguration\nkubernetesVersion: v1.30.0\n",
}

out, err := NewNode(nodeinput)
Expand All @@ -323,6 +324,8 @@ func TestNewJoinNodeCommands(t *testing.T) {
- "echo $(date) ': hello PostKubeadmCommands!'"`

g.Expect(out).To(ContainSubstring(expectedRunCmd))

g.Expect(out).To(ContainSubstring("path: " + ClusterConfigurationPath))
}

func TestOmittableFields(t *testing.T) {
Expand Down
23 changes: 22 additions & 1 deletion bootstrap/kubeadm/internal/cloudinit/node.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,17 @@ limitations under the License.

package cloudinit

import (
bootstrapv1 "sigs.k8s.io/cluster-api/api/bootstrap/kubeadm/v1beta2"
)

const (
// ClusterConfigurationPath is the path where the control plane ClusterConfiguration
// is written for worker nodes. Scripts running as preKubeadmCommands can read this
// file to extract kubernetesVersion, networking, and other cluster-wide settings.
ClusterConfigurationPath = "/run/cluster-api/cluster-configuration.yaml"
)

const (
nodeCloudInit = `{{.Header}}
{{template "files" .WriteFiles}}
Expand Down Expand Up @@ -45,12 +56,22 @@ runcmd:
// NodeInput defines the context to generate a node user data.
type NodeInput struct {
BaseUserData
JoinConfiguration string
JoinConfiguration string
ClusterConfigurationYAML string
}

// NewNode returns the user data string to be used on a node instance.
func NewNode(input *NodeInput) ([]byte, error) {
input.prepare()
input.Header = cloudConfigHeader
if input.ClusterConfigurationYAML != "" {
ccFile := bootstrapv1.File{
Path: ClusterConfigurationPath,
Owner: "root:root",
Permissions: "0644",
Content: input.ClusterConfigurationYAML,
}
input.WriteFiles = append([]bootstrapv1.File{ccFile}, input.WriteFiles...)
}
return generate("Node", nodeCloudInit, input)
}
15 changes: 12 additions & 3 deletions bootstrap/kubeadm/internal/cloudinit/node_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ func TestNewNode(t *testing.T) {
wantErr bool
}{
{
"check for duplicated write_files",
"check for duplicated write_files with ClusterConfiguration",
&NodeInput{
BaseUserData: BaseUserData{
AdditionalFiles: []bootstrapv1.File{
Expand All @@ -45,16 +45,25 @@ func TestNewNode(t *testing.T) {
},
},
},
ClusterConfigurationYAML: "apiVersion: kubeadm.k8s.io/v1beta4\nkind: ClusterConfiguration\nkubernetesVersion: v1.30.0\n",
},
checkWriteFiles("/etc/foo.conf", "/run/kubeadm/kubeadm-join-config.yaml", "/run/cluster-api/placeholder"),
checkWriteFiles(ClusterConfigurationPath, "/etc/foo.conf", "/run/kubeadm/kubeadm-join-config.yaml", "/run/cluster-api/placeholder"),
false,
},
{
"check for existence of /run/kubeadm/kubeadm-join-config.yaml and /run/cluster-api/placeholder",
"check write_files without ClusterConfiguration",
&NodeInput{},
checkWriteFiles("/run/kubeadm/kubeadm-join-config.yaml", "/run/cluster-api/placeholder"),
false,
},
{
"check write_files with ClusterConfiguration",
&NodeInput{
ClusterConfigurationYAML: "apiVersion: kubeadm.k8s.io/v1beta4\nkind: ClusterConfiguration\nkubernetesVersion: v1.30.0\n",
},
checkWriteFiles(ClusterConfigurationPath, "/run/kubeadm/kubeadm-join-config.yaml", "/run/cluster-api/placeholder"),
false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
Expand Down
60 changes: 59 additions & 1 deletion bootstrap/kubeadm/internal/controllers/kubeadmconfig_controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@
"sigs.k8s.io/controller-runtime/pkg/handler"
"sigs.k8s.io/yaml"

bootstrapv1 "sigs.k8s.io/cluster-api/api/bootstrap/kubeadm/v1beta2"

Check failure on line 45 in bootstrap/kubeadm/internal/controllers/kubeadmconfig_controller.go

View workflow job for this annotation

GitHub Actions / lint 

File is not properly formatted (gci)
clusterv1 "sigs.k8s.io/cluster-api/api/core/v1beta2"
"sigs.k8s.io/cluster-api/bootstrap/kubeadm/internal/cloudinit"
"sigs.k8s.io/cluster-api/bootstrap/kubeadm/internal/ignition"
Expand All @@ -51,6 +51,7 @@
"sigs.k8s.io/cluster-api/bootstrap/kubeadm/types/upstream"
bsutil "sigs.k8s.io/cluster-api/bootstrap/util"
"sigs.k8s.io/cluster-api/controllers/clustercache"
controlplanev1 "sigs.k8s.io/cluster-api/api/controlplane/kubeadm/v1beta2"
"sigs.k8s.io/cluster-api/feature"
"sigs.k8s.io/cluster-api/internal/util/taints"
"sigs.k8s.io/cluster-api/util"
Expand All @@ -77,6 +78,7 @@

// +kubebuilder:rbac:groups=bootstrap.cluster.x-k8s.io,resources=kubeadmconfigs;kubeadmconfigs/status;kubeadmconfigs/finalizers,verbs=get;list;watch;create;update;patch;delete
// +kubebuilder:rbac:groups=cluster.x-k8s.io,resources=clusters;clusters/status;machinesets;machines;machines/status;machinepools;machinepools/status,verbs=get;list;watch
// +kubebuilder:rbac:groups=controlplane.cluster.x-k8s.io,resources=kubeadmcontrolplanes;kubeadmcontrolplanes/status,verbs=get;list;watch
// +kubebuilder:rbac:groups="",resources=secrets;configmaps,verbs=get;list;watch;create;update;patch;delete
// +kubebuilder:rbac:groups=core,resources=events,verbs=create;patch

Expand Down Expand Up @@ -685,12 +687,30 @@
return res, nil
}

// Fetch the typed KCP so we can use the control plane version for kubeadm join and
// write the full ClusterConfiguration to disk for worker nodes.
kcp := r.getKCPForJoin(ctx, scope)
kubernetesVersion := scope.ConfigOwner.KubernetesVersion()
if kcp != nil && kcp.Spec.Version != "" {
kubernetesVersion = kcp.Spec.Version
}
parsedVersion, err := semver.ParseTolerant(kubernetesVersion)
if err != nil {
return ctrl.Result{}, errors.Wrapf(err, "failed to parse kubernetes version %q", kubernetesVersion)
}

// Build the ClusterConfiguration YAML to write to disk on the worker node.
var clusterConfigYAML string
if kcp != nil {
cc := kcp.Spec.KubeadmConfigSpec.ClusterConfiguration.DeepCopy()
ad := r.computeAdditionalDataForWorkerJoin(scope.Cluster, kcp)
clusterConfigYAML, err = kubeadmtypes.MarshalClusterConfigurationForVersion(cc, parsedVersion, ad)
if err != nil {
scope.Error(err, "Failed to marshal ClusterConfiguration for worker join")
return ctrl.Result{}, err
}
}

// Add the node uninitialized taint to the list of taints.
// DeepCopy the JoinConfiguration to prevent updating the actual KubeadmConfig.
// Do not modify the KubeadmConfig in etcd as this is a temporary taint that will be dropped after the node
Expand Down Expand Up @@ -779,7 +799,8 @@
KubeadmVerbosity: verbosityFlag,
KubernetesVersion: parsedVersion,
},
JoinConfiguration: joinData,
JoinConfiguration: joinData,
ClusterConfigurationYAML: clusterConfigYAML,
}

var bootstrapJoinData []byte
Expand Down Expand Up @@ -807,6 +828,23 @@
return ctrl.Result{RequeueAfter: r.tokenCheckRefreshOrRotationInterval()}, nil
}

// getKCPForJoin returns the typed KubeadmControlPlane from the cluster's ControlPlaneRef.
// Returns nil if the cluster has no ControlPlaneRef or the KCP cannot be fetched.
// Used for worker join so the bootstrap controller can read the control plane version
// and ClusterConfiguration directly from the KCP spec.
func (r *KubeadmConfigReconciler) getKCPForJoin(ctx context.Context, scope *Scope) *controlplanev1.KubeadmControlPlane {
ref := scope.Cluster.Spec.ControlPlaneRef
if !ref.IsDefined() {
return nil
}
kcp := &controlplanev1.KubeadmControlPlane{}
if err := r.Client.Get(ctx, client.ObjectKey{Namespace: scope.Cluster.Namespace, Name: ref.Name}, kcp); err != nil {
scope.V(4).Info("Could not get KubeadmControlPlane, falling back to machine version", "error", err)
return nil
}
return kcp
}

func (r *KubeadmConfigReconciler) joinControlplane(ctx context.Context, scope *Scope) (ctrl.Result, error) {
scope.Info("Creating BootstrapData for the joining control plane")

Expand Down Expand Up @@ -1339,6 +1377,26 @@
return data
}

// computeAdditionalDataForWorkerJoin builds AdditionalData for serializing the
// ClusterConfiguration that gets written to disk on worker nodes during join.
// It uses the KCP's spec.version as the KubernetesVersion (instead of the
// joining machine's version) and populates networking from the Cluster object.
func (r *KubeadmConfigReconciler) computeAdditionalDataForWorkerJoin(cluster *clusterv1.Cluster, kcp *controlplanev1.KubeadmControlPlane) *upstream.AdditionalData {
data := &upstream.AdditionalData{}
data.ClusterName = ptr.To(cluster.Name)
data.KubernetesVersion = ptr.To(kcp.Spec.Version)
if cluster.Spec.ClusterNetwork.ServiceDomain != "" {
data.DNSDomain = ptr.To(cluster.Spec.ClusterNetwork.ServiceDomain)
}
if len(cluster.Spec.ClusterNetwork.Services.CIDRBlocks) > 0 {
data.ServiceSubnet = ptr.To(cluster.Spec.ClusterNetwork.Services.String())
}
if len(cluster.Spec.ClusterNetwork.Pods.CIDRBlocks) > 0 {
data.PodSubnet = ptr.To(cluster.Spec.ClusterNetwork.Pods.String())
}
return data
}

// storeBootstrapData creates a new secret with the data passed in as input,
// sets the reference in the configuration status and ready to true.
func (r *KubeadmConfigReconciler) storeBootstrapData(ctx context.Context, scope *Scope, data []byte) error {
Expand Down
Loading
Loading