fix: make apply-config work reliably in any Talos state

Fixes #4587

The gist is removing `ApplyConfiguration` sequence and refactoring whole
flow.

We can break down any `ApplyConfiguration` mode into following steps:

* validate incoming config
* apply dynamic config patches (we should get rid of these eventually)
* write down the config to `/state`

If we run in `--immediate` mode, we should also apply configuration
in-memory immediately.

If we run in default mode (apply with reboot), we should actually reboot
the machine (equivalent of `talosctl reboot`), no matter if the
sequencer is stuck in the boot sequence right now.

This fixes mostly apply with reboot mode with following changes:

* machine config is no longer applied in memory (it should be only
applied after a reboot)
* sequence reboot runs with take over, so it overrides sequencer locks
if the machine is stuck in boot sequence and config change is required
to fix that

Signed-off-by: Andrey Smirnov <andrey.smirnov@talos-systems.com>

internal/app/machined/internal/server/v1alpha1/v1alpha1_server.go

@@ -127,47 +127,42 @@ func (s *Server) Register(obj *grpc.Server) {
 func (s *Server) ApplyConfiguration(ctx context.Context, in *machine.ApplyConfigurationRequest) (*machine.ApplyConfigurationResponse, error) {
 	log.Printf("apply config request: immediate %v, on reboot %v", in.Immediate, in.OnReboot)
 
-	applyDynamicConfig := func() ([]byte, error) {
-		cfg, err := s.Controller.Runtime().ValidateConfig(in.GetData())
-		if err != nil {
+	// --immediate
+	if in.Immediate {
+		if err := s.Controller.Runtime().CanApplyImmediate(in.GetData()); err != nil {
 			return nil, err
 		}
+	}
 
-		err = cfg.ApplyDynamicConfig(ctx, s.Controller.Runtime().State().Platform())
-		if err != nil {
-			return nil, err
-		}
+	cfgProvider, err := s.Controller.Runtime().ValidateConfig(in.GetData())
+	if err != nil {
+		return nil, err
+	}
 
-		return cfg.Bytes()
+	err = cfgProvider.ApplyDynamicConfig(ctx, s.Controller.Runtime().State().Platform())
+	if err != nil {
+		return nil, err
+	}
+
+	cfg, err := cfgProvider.Bytes()
+	if err != nil {
+		return nil, err
+	}
+
+	if err := ioutil.WriteFile(constants.ConfigPath, cfg, 0o600); err != nil {
+		return nil, err
 	}
 
 	switch {
 	// --immediate
 	case in.Immediate:
-		if err := s.Controller.Runtime().CanApplyImmediate(in.GetData()); err != nil {
-			return nil, err
-		}
-
-		cfg, err := applyDynamicConfig()
-		if err != nil {
-			return nil, err
-		}
-
 		if err := s.Controller.Runtime().SetConfig(cfg); err != nil {
 			return nil, err
 		}
-
-		if err := ioutil.WriteFile(constants.ConfigPath, in.GetData(), 0o600); err != nil {
-			return nil, err
-		}
-	// default (no flags)
+	// default, no `--on-reboot`
 	case !in.OnReboot:
-		if err := s.Controller.Runtime().SetConfig(in.GetData()); err != nil {
-			return nil, err
-		}
-
 		go func() {
-			if err := s.Controller.Run(context.Background(), runtime.SequenceApplyConfiguration, in); err != nil {
+			if err := s.Controller.Run(context.Background(), runtime.SequenceReboot, nil, runtime.WithTakeover()); err != nil {
 				if !runtime.IsRebootError(err) {
 					log.Println("apply configuration failed:", err)
 				}
@@ -177,16 +172,6 @@ func (s *Server) ApplyConfiguration(ctx context.Context, in *machine.ApplyConfig
 				}
 			}
 		}()
-	// --no-reboot
-	case in.OnReboot:
-		cfg, err := applyDynamicConfig()
-		if err != nil {
-			return nil, err
-		}
-
-		if err = ioutil.WriteFile(constants.ConfigPath, cfg, 0o600); err != nil {
-			return nil, err
-		}
 	}
 
 	return &machine.ApplyConfigurationResponse{

internal/app/machined/pkg/runtime/sequencer.go

@@ -14,10 +14,8 @@ import (
 type Sequence int
 
 const (
-	// SequenceApplyConfiguration is the apply configuration sequence.
-	SequenceApplyConfiguration Sequence = iota
 	// SequenceBoot is the boot sequence.
-	SequenceBoot
+	SequenceBoot Sequence = iota
 	// SequenceBootstrap is the boot sequence.
 	SequenceBootstrap
 	// SequenceInitialize is the initialize sequence.
@@ -39,31 +37,28 @@ const (
 )
 
 const (
-	applyConfiguration = "applyConfiguration"
-	boot               = "boot"
-	bootstrap          = "bootstrap"
-	initialize         = "initialize"
-	install            = "install"
-	shutdown           = "shutdown"
-	upgrade            = "upgrade"
-	stageUpgrade       = "stageUpgrade"
-	reset              = "reset"
-	reboot             = "reboot"
-	noop               = "noop"
+	boot         = "boot"
+	bootstrap    = "bootstrap"
+	initialize   = "initialize"
+	install      = "install"
+	shutdown     = "shutdown"
+	upgrade      = "upgrade"
+	stageUpgrade = "stageUpgrade"
+	reset        = "reset"
+	reboot       = "reboot"
+	noop         = "noop"
 )
 
 // String returns the string representation of a `Sequence`.
 func (s Sequence) String() string {
-	return [...]string{applyConfiguration, boot, bootstrap, initialize, install, shutdown, upgrade, stageUpgrade, reset, reboot, noop}[s]
+	return [...]string{boot, bootstrap, initialize, install, shutdown, upgrade, stageUpgrade, reset, reboot, noop}[s]
 }
 
 // ParseSequence returns a `Sequence` that matches the specified string.
 //
 //nolint:gocyclo
 func ParseSequence(s string) (seq Sequence, err error) {
 	switch s {
-	case applyConfiguration:
-		seq = SequenceApplyConfiguration
 	case boot:
 		seq = SequenceBoot
 	case bootstrap:
@@ -107,7 +102,6 @@ type PartitionTarget interface {
 // Sequencer describes the set of sequences required for the lifecycle
 // management of the operating system.
 type Sequencer interface {
-	ApplyConfiguration(Runtime, *machine.ApplyConfigurationRequest) []Phase
 	Boot(Runtime) []Phase
 	Bootstrap(Runtime) []Phase
 	Initialize(Runtime) []Phase

internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_controller.go

@@ -382,17 +382,6 @@ func (c *Controller) phases(seq runtime.Sequence, data interface{}) ([]runtime.P
 	var phases []runtime.Phase
 
 	switch seq {
-	case runtime.SequenceApplyConfiguration:
-		var (
-			in *machine.ApplyConfigurationRequest
-			ok bool
-		)
-
-		if in, ok = data.(*machine.ApplyConfigurationRequest); !ok {
-			return nil, runtime.ErrInvalidSequenceData
-		}
-
-		phases = c.s.ApplyConfiguration(c.r, in)
 	case runtime.SequenceBoot:
 		phases = c.s.Boot(c.r)
 	case runtime.SequenceBootstrap:

internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_sequencer.go

@@ -45,35 +45,6 @@ func (p PhaseList) AppendList(list PhaseList) PhaseList {
 	return append(p, list...)
 }
 
-// ApplyConfiguration defines a sequence which applies a new machine configuration to the node, rebooting to make it active.
-func (*Sequencer) ApplyConfiguration(r runtime.Runtime, req *machineapi.ApplyConfigurationRequest) []runtime.Phase {
-	phases := PhaseList{}
-
-	phases = phases.Append(
-		"saveStateEncryptionConfig",
-		SaveStateEncryptionConfig,
-	).Append(
-		"mountState",
-		MountStatePartition,
-	).Append(
-		"saveConfig",
-		SaveConfig,
-	).Append(
-		"unmountState",
-		UnmountStatePartition,
-	).Append(
-		"cleanup",
-		StopAllPods,
-	).AppendList(
-		stopAllPhaselist(r, true),
-	).Append(
-		"reboot",
-		Reboot,
-	)
-
-	return phases
-}
-
 // Initialize is the initialize sequence. The primary goals of this sequence is
 // to load the config and enforce kernel security requirements.
 func (*Sequencer) Initialize(r runtime.Runtime) []runtime.Phase {