Skip to content

Commit

Permalink
feat: can disable controlmanager and scheduler
Browse files Browse the repository at this point in the history 
This PR solves problems:
1. You want to run controlplane from etcd-backup, and make
   changes/investigation and be sure that controlplane
   does no change resources.
2. Run controlplane in less memory only for quorum.

Signed-off-by: Serge Logvinov <serge.logvinov@sinextra.dev>
Co-authored-by: Andrey Smirnov <andrey.smirnov@talos-systems.com>
Signed-off-by: Andrey Smirnov <andrey.smirnov@talos-systems.com>
  • Loading branch information
@sergelogvinov @smira
sergelogvinov and smira committed Dec 2, 2021
1 parent 7f99222 commit 73293bc
Show file tree
Hide file tree
Showing 11 changed files with 602 additions and 153 deletions.
2 changes: 2 additions & 0 deletions internal/app/machined/pkg/controllers/config/k8s_control_plane.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -165,6 +165,7 @@ func (ctrl *K8sControlPlaneController) manageControllerManagerConfig(ctx context

return r.Modify(ctx, config.NewK8sControlPlaneControllerManager(), func(r resource.Resource) error {
r.(*config.K8sControlPlane).SetControllerManager(config.K8sControlPlaneControllerManagerSpec{
Enabled: !cfgProvider.Machine().Controlplane().ControllerManager().Disabled(),
Image: cfgProvider.Cluster().ControllerManager().Image(),
CloudProvider: cloudProvider,
PodCIDRs: cfgProvider.Cluster().Network().PodCIDRs(),
Expand All @@ -180,6 +181,7 @@ func (ctrl *K8sControlPlaneController) manageControllerManagerConfig(ctx context
func (ctrl *K8sControlPlaneController) manageSchedulerConfig(ctx context.Context, r controller.Runtime, logger *zap.Logger, cfgProvider talosconfig.Provider) error {
return r.Modify(ctx, config.NewK8sControlPlaneScheduler(), func(r resource.Resource) error {
r.(*config.K8sControlPlane).SetScheduler(config.K8sControlPlaneSchedulerSpec{
Enabled: !cfgProvider.Machine().Controlplane().Scheduler().Disabled(),
Image: cfgProvider.Cluster().Scheduler().Image(),
ExtraArgs: cfgProvider.Cluster().Scheduler().ExtraArgs(),
ExtraVolumes: convertVolumes(cfgProvider.Cluster().Scheduler().ExtraVolumes()),
Expand Down
57 changes: 46 additions & 11 deletions internal/app/machined/pkg/controllers/k8s/control_plane_static_pod.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -114,8 +114,10 @@ func (ctrl *ControlPlaneStaticPodController) Run(ctx context.Context, r controll

secretsVersion := secretsStatusResource.(*k8s.SecretsStatus).TypedSpec().Version

touchedIDs := map[string]struct{}{}

for _, pod := range []struct {
f func(context.Context, controller.Runtime, *zap.Logger, *config.K8sControlPlane, string) error
f func(context.Context, controller.Runtime, *zap.Logger, *config.K8sControlPlane, string) (string, error)
id resource.ID
}{
{
Expand All @@ -140,9 +142,33 @@ func (ctrl *ControlPlaneStaticPodController) Run(ctx context.Context, r controll
return fmt.Errorf("error getting control plane config: %w", err)
}

if err = pod.f(ctx, r, logger, res.(*config.K8sControlPlane), secretsVersion); err != nil {
var podID string

if podID, err = pod.f(ctx, r, logger, res.(*config.K8sControlPlane), secretsVersion); err != nil {
return fmt.Errorf("error updating static pod for %q: %w", pod.id, err)
}

if podID != "" {
touchedIDs[podID] = struct{}{}
}
}

// clean up static pods which haven't been touched
{
list, err := r.List(ctx, resource.NewMetadata(k8s.ControlPlaneNamespaceName, k8s.StaticPodType, "", resource.VersionUndefined))
if err != nil {
return err
}

for _, res := range list.Items {
if _, ok := touchedIDs[res.Metadata().ID()]; ok {
continue
}

if err = r.Destroy(ctx, res.Metadata()); err != nil {
return err
}
}
}
}
}
Expand Down Expand Up @@ -195,7 +221,8 @@ func volumes(volumes []config.K8sExtraVolume) []v1.Volume {
return result
}

func (ctrl *ControlPlaneStaticPodController) manageAPIServer(ctx context.Context, r controller.Runtime, logger *zap.Logger, configResource *config.K8sControlPlane, secretsVersion string) error {
func (ctrl *ControlPlaneStaticPodController) manageAPIServer(ctx context.Context, r controller.Runtime, logger *zap.Logger,
configResource *config.K8sControlPlane, secretsVersion string) (string, error) {
cfg := configResource.APIServer()

enabledAdmissionPlugins := []string{"NodeRestriction"}
Expand Down Expand Up @@ -275,12 +302,12 @@ func (ctrl *ControlPlaneStaticPodController) manageAPIServer(ctx context.Context
}

if err := builder.Merge(cfg.ExtraArgs, argsbuilder.WithMergePolicies(mergePolicies)); err != nil {
return err
return "", err
}

args = append(args, builder.Args()...)

return r.Modify(ctx, k8s.NewStaticPod(k8s.ControlPlaneNamespaceName, "kube-apiserver"), func(r resource.Resource) error {
return config.K8sControlPlaneAPIServerID, r.Modify(ctx, k8s.NewStaticPod(k8s.ControlPlaneNamespaceName, config.K8sControlPlaneAPIServerID), func(r resource.Resource) error {
return k8sadapter.StaticPod(r.(*k8s.StaticPod)).SetPod(&v1.Pod{
TypeMeta: metav1.TypeMeta{
APIVersion: "v1",
Expand Down Expand Up @@ -351,9 +378,13 @@ func (ctrl *ControlPlaneStaticPodController) manageAPIServer(ctx context.Context
}

func (ctrl *ControlPlaneStaticPodController) manageControllerManager(ctx context.Context, r controller.Runtime,
logger *zap.Logger, configResource *config.K8sControlPlane, secretsVersion string) error {
logger *zap.Logger, configResource *config.K8sControlPlane, secretsVersion string) (string, error) {
cfg := configResource.ControllerManager()

if !cfg.Enabled {
return "", nil
}

args := []string{
"/usr/local/bin/kube-controller-manager",
"--use-service-account-credentials",
Expand Down Expand Up @@ -395,13 +426,13 @@ func (ctrl *ControlPlaneStaticPodController) manageControllerManager(ctx context
}

if err := builder.Merge(cfg.ExtraArgs, argsbuilder.WithMergePolicies(mergePolicies)); err != nil {
return err
return "", err
}

args = append(args, builder.Args()...)

//nolint:dupl
return r.Modify(ctx, k8s.NewStaticPod(k8s.ControlPlaneNamespaceName, "kube-controller-manager"), func(r resource.Resource) error {
return config.K8sControlPlaneControllerManagerID, r.Modify(ctx, k8s.NewStaticPod(k8s.ControlPlaneNamespaceName, config.K8sControlPlaneControllerManagerID), func(r resource.Resource) error {
return k8sadapter.StaticPod(r.(*k8s.StaticPod)).SetPod(&v1.Pod{
TypeMeta: metav1.TypeMeta{
APIVersion: "v1",
Expand Down Expand Up @@ -474,9 +505,13 @@ func (ctrl *ControlPlaneStaticPodController) manageControllerManager(ctx context
}

func (ctrl *ControlPlaneStaticPodController) manageScheduler(ctx context.Context, r controller.Runtime,
logger *zap.Logger, configResource *config.K8sControlPlane, secretsVersion string) error {
logger *zap.Logger, configResource *config.K8sControlPlane, secretsVersion string) (string, error) {
cfg := configResource.Scheduler()

if !cfg.Enabled {
return "", nil
}

args := []string{
"/usr/local/bin/kube-scheduler",
}
Expand All @@ -499,13 +534,13 @@ func (ctrl *ControlPlaneStaticPodController) manageScheduler(ctx context.Context
}

if err := builder.Merge(cfg.ExtraArgs, argsbuilder.WithMergePolicies(mergePolicies)); err != nil {
return err
return "", err
}

args = append(args, builder.Args()...)

//nolint:dupl
return r.Modify(ctx, k8s.NewStaticPod(k8s.ControlPlaneNamespaceName, "kube-scheduler"), func(r resource.Resource) error {
return config.K8sControlPlaneSchedulerID, r.Modify(ctx, k8s.NewStaticPod(k8s.ControlPlaneNamespaceName, config.K8sControlPlaneSchedulerID), func(r resource.Resource) error {
return k8sadapter.StaticPod(r.(*k8s.StaticPod)).SetPod(&v1.Pod{
TypeMeta: metav1.TypeMeta{
APIVersion: "v1",
Expand Down
47 changes: 47 additions & 0 deletions internal/app/machined/pkg/controllers/k8s/control_plane_static_pod_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -312,6 +312,53 @@ func (suite *ControlPlaneStaticPodSuite) TestReconcileExtraArgs() {
}
}

func (suite *ControlPlaneStaticPodSuite) TestControlPlaneStaticPodsExeptScheduler() {
secretStatus := k8s.NewSecretsStatus(k8s.ControlPlaneNamespaceName, k8s.StaticPodSecretsStaticPodID)
configAPIServer := config.NewK8sControlPlaneAPIServer()
configControllerManager := config.NewK8sControlPlaneControllerManager()
configScheduler := config.NewK8sControlPlaneScheduler()

suite.Require().NoError(suite.state.Create(suite.ctx, secretStatus))
suite.Require().NoError(suite.state.Create(suite.ctx, configAPIServer))
suite.Require().NoError(suite.state.Create(suite.ctx, configControllerManager))
suite.Require().NoError(suite.state.Create(suite.ctx, configScheduler))

suite.Assert().NoError(retry.Constant(10*time.Second, retry.WithUnits(100*time.Millisecond)).Retry(
func() error {
return suite.assertControlPlaneStaticPods(
[]string{
"kube-apiserver",
"kube-controller-manager",
"kube-scheduler",
},
)
},
))

// flip enabled to disable scheduler
_, err := suite.state.UpdateWithConflicts(suite.ctx, configScheduler.Metadata(), func(r resource.Resource) error {
spec := r.(*config.K8sControlPlane).Scheduler()
spec.Enabled = false
r.(*config.K8sControlPlane).SetScheduler(spec)

return nil
})
suite.Require().NoError(err)

configScheduler.SetScheduler(config.K8sControlPlaneSchedulerSpec{Enabled: false})

suite.Assert().NoError(retry.Constant(10*time.Second, retry.WithUnits(100*time.Millisecond)).Retry(
func() error {
return suite.assertControlPlaneStaticPods(
[]string{
"kube-apiserver",
"kube-controller-manager",
},
)
},
))
}

func (suite *ControlPlaneStaticPodSuite) TearDownTest() {
suite.T().Log("tear down")

Expand Down
2 changes: 2 additions & 0 deletions internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_runtime.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,7 @@ func (r *Runtime) CanApplyImmediate(b []byte) error {
// * .machine.network
// * .machine.sysctls
// * .machine.logging
// * .machine.controlplane
newConfig.ConfigDebug = currentConfig.ConfigDebug
newConfig.ClusterConfig = currentConfig.ClusterConfig

Expand All @@ -119,6 +120,7 @@ func (r *Runtime) CanApplyImmediate(b []byte) error {
newConfig.MachineConfig.MachineNetwork = currentConfig.MachineConfig.MachineNetwork
newConfig.MachineConfig.MachineSysctls = currentConfig.MachineConfig.MachineSysctls
newConfig.MachineConfig.MachineLogging = currentConfig.MachineConfig.MachineLogging
newConfig.MachineConfig.MachineControlPlane = currentConfig.MachineConfig.MachineControlPlane
}

if !reflect.DeepEqual(currentConfig, newConfig) {
Expand Down
20 changes: 20 additions & 0 deletions pkg/machinery/config/provider.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ type MachineConfig interface {
Env() Env
Files() ([]File, error)
Type() machine.Type
Controlplane() MachineControlPlane
Kubelet() Kubelet
Sysctls() map[string]string
Registries() Registries
Expand Down Expand Up @@ -96,6 +97,25 @@ type Security interface {
CertSANs() []string
}

// MachineControlPlane defines the requirements for a config that pertains to Controlplane
// related options.
type MachineControlPlane interface {
ControllerManager() MachineControllerManager
Scheduler() MachineScheduler
}

// MachineControllerManager defines the requirements for a config that pertains to ControllerManager
// related options.
type MachineControllerManager interface {
Disabled() bool
}

// MachineScheduler defines the requirements for a config that pertains to Scheduler
// related options.
type MachineScheduler interface {
Disabled() bool
}

// MachineNetwork defines the requirements for a config that pertains to network
// related options.
type MachineNetwork interface {
Expand Down
37 changes: 37 additions & 0 deletions pkg/machinery/config/types/v1alpha1/v1alpha1_provider.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -194,6 +194,43 @@ func (m *MachineConfig) Time() config.Time {
return m.MachineTime
}

// Controlplane implements the config.Provider interface.
func (m *MachineConfig) Controlplane() config.MachineControlPlane {
if m.MachineControlPlane == nil {
return &MachineControlPlaneConfig{}
}

return m.MachineControlPlane
}

// ControllerManager implements the config.Provider interface.
func (m *MachineControlPlaneConfig) ControllerManager() config.MachineControllerManager {
if m.MachineControllerManager == nil {
return &MachineControllerManagerConfig{}
}

return m.MachineControllerManager
}

// Scheduler implements the config.Provider interface.
func (m *MachineControlPlaneConfig) Scheduler() config.MachineScheduler {
if m.MachineScheduler == nil {
return &MachineSchedulerConfig{}
}

return m.MachineScheduler
}

// Disabled implements the config.Provider interface.
func (m *MachineControllerManagerConfig) Disabled() bool {
return m.MachineControllerManagerDisabled
}

// Disabled implements the config.Provider interface.
func (m *MachineSchedulerConfig) Disabled() bool {
return m.MachineSchedulerDisabled
}

// Kubelet implements the config.Provider interface.
func (m *MachineConfig) Kubelet() config.Kubelet {
if m.MachineKubelet == nil {
Expand Down
35 changes: 35 additions & 0 deletions pkg/machinery/config/types/v1alpha1/v1alpha1_types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -134,6 +134,11 @@ var (
Key: []byte("LS0tLS1CRUdJTiBFRDI1NTE5IFBSSVZBVEUgS0VZLS0tLS0KTUM..."),
}

machineControlplaneExample = &MachineControlPlaneConfig{
MachineControllerManager: &MachineControllerManagerConfig{},
MachineScheduler: &MachineSchedulerConfig{MachineSchedulerDisabled: true},
}

machineKubeletExample = &KubeletConfig{
KubeletImage: (&KubeletConfig{}).Image(),
KubeletExtraArgs: map[string]string{
Expand Down Expand Up @@ -582,6 +587,12 @@ type MachineConfig struct {
// value: '[]string{"10.0.0.10", "172.16.0.10", "192.168.0.10"}'
MachineCertSANs []string `yaml:"certSANs"`
// description: |
// Provides machine specific contolplane configuration options.
// examples:
// - name: ControlPlane definition example.
// value: machineControlplaneExample
MachineControlPlane *MachineControlPlaneConfig `yaml:"controlPlane,omitempty"`
// description: |
// Used to provide additional options to the kubelet.
// examples:
// - name: Kubelet definition example.
Expand Down Expand Up @@ -847,6 +858,30 @@ func (in *ExtraMount) DeepCopy() *ExtraMount {
return out
}

// MachineControlPlaneConfig machine specific configuration options.
type MachineControlPlaneConfig struct {
// description: |
// Controller manager machine specific configuration options.
MachineControllerManager *MachineControllerManagerConfig `yaml:"controllerManager,omitempty"`
// description: |
// Scheduler machine specific configuration options.
MachineScheduler *MachineSchedulerConfig `yaml:"scheduler,omitempty"`
}

// MachineControllerManagerConfig represents the machine specific ControllerManager config values.
type MachineControllerManagerConfig struct {
// description: |
// Disable kube-controller-manager on the node.
MachineControllerManagerDisabled bool `yaml:"disabled"`
}

// MachineSchedulerConfig represents the machine specific Scheduler config values.
type MachineSchedulerConfig struct {
// description: |
// Disable kube-scheduler on the node.
MachineSchedulerDisabled bool `yaml:"disabled"`
}

// KubeletConfig represents the kubelet config values.
type KubeletConfig struct {
// description: |
Expand Down

0 comments on commit 73293bc

Please sign in to comment.