diff --git a/.mockery.yaml b/.mockery.yaml index 53efe9faabd..8d8a896feb3 100644 --- a/.mockery.yaml +++ b/.mockery.yaml @@ -18,6 +18,7 @@ packages: interfaces: WatcherHelper: {} watcherGrappler: {} + availableRollbacksSource: {} github.com/elastic/elastic-agent/internal/pkg/agent/cmd: interfaces: agentWatcher: {} diff --git a/internal/pkg/agent/application/application.go b/internal/pkg/agent/application/application.go index 1c9c53fe409..825b078cceb 100644 --- a/internal/pkg/agent/application/application.go +++ b/internal/pkg/agent/application/application.go @@ -64,7 +64,7 @@ func New( fleetInitTimeout time.Duration, disableMonitoring bool, override CfgOverrider, - initialUpgradeDetails *details.Details, + initialUpdateMarker *upgrade.UpdateMarker, modifiers ...component.PlatformModifier, ) (*coordinator.Coordinator, coordinator.ConfigManager, composable.Controller, error) { @@ -129,7 +129,9 @@ func New( // monitoring is not supported in bootstrap mode https://github.com/elastic/elastic-agent/issues/1761 isMonitoringSupported := !disableMonitoring && cfg.Settings.V1MonitoringEnabled - upgrader, err := upgrade.NewUpgrader(log, cfg.Settings.DownloadConfig, cfg.Settings.Upgrade, agentInfo, new(upgrade.AgentWatcherHelper)) + + availableRollbacksSource := upgrade.NewTTLMarkerRegistry(log, paths.Top()) + upgrader, err := upgrade.NewUpgrader(log, cfg.Settings.DownloadConfig, cfg.Settings.Upgrade, agentInfo, new(upgrade.AgentWatcherHelper), availableRollbacksSource) if err != nil { return nil, nil, nil, fmt.Errorf("failed to create upgrader: %w", err) } @@ -153,6 +155,12 @@ func New( return nil, nil, nil, fmt.Errorf("failed to initialize runtime manager: %w", err) } + // prepare initialUpgradeDetails for injecting it in coordinator later on + var initialUpgradeDetails *details.Details + if initialUpdateMarker != nil && initialUpdateMarker.Details != nil { + initialUpgradeDetails = initialUpdateMarker.Details + } + var configMgr coordinator.ConfigManager var managed *managedConfigManager var compModifiers = []coordinator.ComponentsModifier{InjectAPMConfig} diff --git a/internal/pkg/agent/application/coordinator/coordinator_unit_test.go b/internal/pkg/agent/application/coordinator/coordinator_unit_test.go index 6dd014761a6..559ad802c7f 100644 --- a/internal/pkg/agent/application/coordinator/coordinator_unit_test.go +++ b/internal/pkg/agent/application/coordinator/coordinator_unit_test.go @@ -467,7 +467,8 @@ func TestCoordinatorReportsInvalidPolicy(t *testing.T) { } }() - upgradeMgr, err := upgrade.NewUpgrader(log, &artifact.Config{}, nil, &info.AgentInfo{}, new(upgrade.AgentWatcherHelper)) + tmpDir := t.TempDir() + upgradeMgr, err := upgrade.NewUpgrader(log, &artifact.Config{}, nil, &info.AgentInfo{}, new(upgrade.AgentWatcherHelper), upgrade.NewTTLMarkerRegistry(nil, tmpDir)) require.NoError(t, err, "errored when creating a new upgrader") // Channels have buffer length 1, so we don't have to run on multiple diff --git a/internal/pkg/agent/application/upgrade/mocks.go b/internal/pkg/agent/application/upgrade/mocks.go index 1820d1a3c90..5ef4993d084 100644 --- a/internal/pkg/agent/application/upgrade/mocks.go +++ b/internal/pkg/agent/application/upgrade/mocks.go @@ -339,6 +339,139 @@ func (_c *MockWatcherHelper_WaitForWatcher_Call) RunAndReturn(run func(ctx conte return _c } +// newMockAvailableRollbacksSource creates a new instance of mockAvailableRollbacksSource. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func newMockAvailableRollbacksSource(t interface { + mock.TestingT + Cleanup(func()) +}) *mockAvailableRollbacksSource { + mock := &mockAvailableRollbacksSource{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} + +// mockAvailableRollbacksSource is an autogenerated mock type for the availableRollbacksSource type +type mockAvailableRollbacksSource struct { + mock.Mock +} + +type mockAvailableRollbacksSource_Expecter struct { + mock *mock.Mock +} + +func (_m *mockAvailableRollbacksSource) EXPECT() *mockAvailableRollbacksSource_Expecter { + return &mockAvailableRollbacksSource_Expecter{mock: &_m.Mock} +} + +// Get provides a mock function for the type mockAvailableRollbacksSource +func (_mock *mockAvailableRollbacksSource) Get() (map[string]TTLMarker, error) { + ret := _mock.Called() + + if len(ret) == 0 { + panic("no return value specified for Get") + } + + var r0 map[string]TTLMarker + var r1 error + if returnFunc, ok := ret.Get(0).(func() (map[string]TTLMarker, error)); ok { + return returnFunc() + } + if returnFunc, ok := ret.Get(0).(func() map[string]TTLMarker); ok { + r0 = returnFunc() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(map[string]TTLMarker) + } + } + if returnFunc, ok := ret.Get(1).(func() error); ok { + r1 = returnFunc() + } else { + r1 = ret.Error(1) + } + return r0, r1 +} + +// mockAvailableRollbacksSource_Get_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Get' +type mockAvailableRollbacksSource_Get_Call struct { + *mock.Call +} + +// Get is a helper method to define mock.On call +func (_e *mockAvailableRollbacksSource_Expecter) Get() *mockAvailableRollbacksSource_Get_Call { + return &mockAvailableRollbacksSource_Get_Call{Call: _e.mock.On("Get")} +} + +func (_c *mockAvailableRollbacksSource_Get_Call) Run(run func()) *mockAvailableRollbacksSource_Get_Call { + _c.Call.Run(func(args mock.Arguments) { + run() + }) + return _c +} + +func (_c *mockAvailableRollbacksSource_Get_Call) Return(stringToTTLMarker map[string]TTLMarker, err error) *mockAvailableRollbacksSource_Get_Call { + _c.Call.Return(stringToTTLMarker, err) + return _c +} + +func (_c *mockAvailableRollbacksSource_Get_Call) RunAndReturn(run func() (map[string]TTLMarker, error)) *mockAvailableRollbacksSource_Get_Call { + _c.Call.Return(run) + return _c +} + +// Set provides a mock function for the type mockAvailableRollbacksSource +func (_mock *mockAvailableRollbacksSource) Set(stringToTTLMarker map[string]TTLMarker) error { + ret := _mock.Called(stringToTTLMarker) + + if len(ret) == 0 { + panic("no return value specified for Set") + } + + var r0 error + if returnFunc, ok := ret.Get(0).(func(map[string]TTLMarker) error); ok { + r0 = returnFunc(stringToTTLMarker) + } else { + r0 = ret.Error(0) + } + return r0 +} + +// mockAvailableRollbacksSource_Set_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Set' +type mockAvailableRollbacksSource_Set_Call struct { + *mock.Call +} + +// Set is a helper method to define mock.On call +// - stringToTTLMarker map[string]TTLMarker +func (_e *mockAvailableRollbacksSource_Expecter) Set(stringToTTLMarker interface{}) *mockAvailableRollbacksSource_Set_Call { + return &mockAvailableRollbacksSource_Set_Call{Call: _e.mock.On("Set", stringToTTLMarker)} +} + +func (_c *mockAvailableRollbacksSource_Set_Call) Run(run func(stringToTTLMarker map[string]TTLMarker)) *mockAvailableRollbacksSource_Set_Call { + _c.Call.Run(func(args mock.Arguments) { + var arg0 map[string]TTLMarker + if args[0] != nil { + arg0 = args[0].(map[string]TTLMarker) + } + run( + arg0, + ) + }) + return _c +} + +func (_c *mockAvailableRollbacksSource_Set_Call) Return(err error) *mockAvailableRollbacksSource_Set_Call { + _c.Call.Return(err) + return _c +} + +func (_c *mockAvailableRollbacksSource_Set_Call) RunAndReturn(run func(stringToTTLMarker map[string]TTLMarker) error) *mockAvailableRollbacksSource_Set_Call { + _c.Call.Return(run) + return _c +} + // newMockWatcherGrappler creates a new instance of mockWatcherGrappler. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. // The first argument is typically a *testing.T value. func newMockWatcherGrappler(t interface { diff --git a/internal/pkg/agent/application/upgrade/rollback_test.go b/internal/pkg/agent/application/upgrade/rollback_test.go index 9bb2f113a66..eab955d663a 100644 --- a/internal/pkg/agent/application/upgrade/rollback_test.go +++ b/internal/pkg/agent/application/upgrade/rollback_test.go @@ -769,6 +769,6 @@ func createUpdateMarker(t *testing.T, log *logger.Logger, topDir, newAgentVersio time.Now(), newAgentInstall, oldAgentInstall, - nil, nil, disableRollbackWindow) + nil, nil, nil) require.NoError(t, err, "error writing fake update marker") } diff --git a/internal/pkg/agent/application/upgrade/step_mark.go b/internal/pkg/agent/application/upgrade/step_mark.go index 3a279a631e9..7f93ff43a27 100644 --- a/internal/pkg/agent/application/upgrade/step_mark.go +++ b/internal/pkg/agent/application/upgrade/step_mark.go @@ -23,10 +23,9 @@ import ( const markerFilename = ".update-marker" const disableRollbackWindow = time.Duration(0) -// RollbackAvailable identifies an elastic-agent install available for rollback -type RollbackAvailable struct { +// TTLMarker marks an elastic-agent install available for rollback +type TTLMarker struct { Version string `json:"version" yaml:"version"` - Home string `json:"home" yaml:"home"` ValidUntil time.Time `json:"valid_until" yaml:"valid_until"` } @@ -55,7 +54,7 @@ type UpdateMarker struct { Details *details.Details `json:"details,omitempty" yaml:"details,omitempty"` - RollbacksAvailable []RollbackAvailable `json:"rollbacks_available,omitempty" yaml:"rollbacks_available,omitempty"` + RollbacksAvailable map[string]TTLMarker `json:"rollbacks_available,omitempty" yaml:"rollbacks_available,omitempty"` } // GetActionID returns the Fleet Action ID associated with the @@ -112,7 +111,7 @@ type updateMarkerSerializer struct { Acked bool `yaml:"acked"` Action *MarkerActionUpgrade `yaml:"action"` Details *details.Details `yaml:"details"` - RollbacksAvailable []RollbackAvailable `yaml:"rollbacks_available,omitempty"` + RollbacksAvailable map[string]TTLMarker `yaml:"rollbacks_available,omitempty"` } func newMarkerSerializer(m *UpdateMarker) *updateMarkerSerializer { @@ -142,35 +141,23 @@ type updateActiveCommitFunc func(log *logger.Logger, topDirPath, hash string, wr // markUpgrade marks update happened so we can handle grace period func markUpgradeProvider(updateActiveCommit updateActiveCommitFunc, writeFile writeFileFunc) markUpgradeFunc { - return func(log *logger.Logger, dataDirPath string, updatedOn time.Time, agent, previousAgent agentInstall, action *fleetapi.ActionUpgrade, upgradeDetails *details.Details, rollbackWindow time.Duration) error { + return func(log *logger.Logger, dataDirPath string, updatedOn time.Time, agent, previousAgent agentInstall, action *fleetapi.ActionUpgrade, upgradeDetails *details.Details, availableRollbacks map[string]TTLMarker) error { if len(previousAgent.hash) > hashLen { previousAgent.hash = previousAgent.hash[:hashLen] } marker := &UpdateMarker{ - Version: agent.version, - Hash: agent.hash, - VersionedHome: agent.versionedHome, - UpdatedOn: updatedOn, - PrevVersion: previousAgent.version, - PrevHash: previousAgent.hash, - PrevVersionedHome: previousAgent.versionedHome, - Action: action, - Details: upgradeDetails, - } - - if rollbackWindow > disableRollbackWindow && agent.parsedVersion != nil && !agent.parsedVersion.Less(*Version_9_2_0_SNAPSHOT) { - // if we have a not empty rollback window, write the prev version in the rollbacks_available field - // we also need to check the destination version because the manual rollback and delayed cleanup will be - // handled by that version of agent, so it needs to be recent enough - marker.RollbacksAvailable = []RollbackAvailable{ - { - Version: previousAgent.version, - Home: previousAgent.versionedHome, - ValidUntil: updatedOn.Add(rollbackWindow), - }, - } + Version: agent.version, + Hash: agent.hash, + VersionedHome: agent.versionedHome, + UpdatedOn: updatedOn, + PrevVersion: previousAgent.version, + PrevHash: previousAgent.hash, + PrevVersionedHome: previousAgent.versionedHome, + Action: action, + Details: upgradeDetails, + RollbacksAvailable: availableRollbacks, } markerBytes, err := yaml.Marshal(newMarkerSerializer(marker)) diff --git a/internal/pkg/agent/application/upgrade/step_mark_test.go b/internal/pkg/agent/application/upgrade/step_mark_test.go index 6d53d75330b..9daed169e58 100644 --- a/internal/pkg/agent/application/upgrade/step_mark_test.go +++ b/internal/pkg/agent/application/upgrade/step_mark_test.go @@ -86,14 +86,15 @@ func TestMarkUpgrade(t *testing.T) { var parsed920SNAPSHOT = agtversion.NewParsedSemVer(9, 2, 0, "SNAPSHOT", "") // fix a timestamp (truncated to the second because of loss of precision during marshalling/unmarshalling) updatedOnNow := time.Now().UTC().Truncate(time.Second) + twentyFourHoursFromNow := updatedOnNow.Add(24 * time.Hour) type args struct { - updatedOn time.Time - currentAgent agentInstall - previousAgent agentInstall - action *fleetapi.ActionUpgrade - details *details.Details - rollbackWindow time.Duration + updatedOn time.Time + currentAgent agentInstall + previousAgent agentInstall + action *fleetapi.ActionUpgrade + details *details.Details + availableRollbacks map[string]TTLMarker } type workingDirHook func(t *testing.T, dataDir string) @@ -130,14 +131,14 @@ func TestMarkUpgrade(t *testing.T) { hash: "prvagt", versionedHome: filepath.Join("data", "elastic-agent-1.2.3-SNAPSHOT-prvagt"), }, - action: nil, - details: details.NewDetails("4.5.6-SNAPSHOT", details.StateReplacing, ""), - rollbackWindow: 0, + action: nil, + details: details.NewDetails("4.5.6-SNAPSHOT", details.StateReplacing, ""), + availableRollbacks: nil, }, wantErr: assert.Error, }, { - name: "no rollback window specified - no available rollbacks", + name: "no rollbacks specified in input - no available rollbacks in marker", args: args{ updatedOn: updatedOnNow, currentAgent: agentInstall{ @@ -152,9 +153,9 @@ func TestMarkUpgrade(t *testing.T) { hash: "prvagt", versionedHome: filepath.Join("data", "elastic-agent-1.2.3-SNAPSHOT-prvagt"), }, - action: nil, - details: details.NewDetails("4.5.6-SNAPSHOT", details.StateReplacing, ""), - rollbackWindow: 0, + action: nil, + details: details.NewDetails("4.5.6-SNAPSHOT", details.StateReplacing, ""), + availableRollbacks: nil, }, wantErr: assert.NoError, assertAfterMark: func(t *testing.T, dataDir string) { @@ -182,51 +183,7 @@ func TestMarkUpgrade(t *testing.T) { }, }, { - name: "rollback window specified but new version is too low - no rollbacks", - args: args{ - updatedOn: updatedOnNow, - currentAgent: agentInstall{ - parsedVersion: parsed456SNAPSHOT, - version: "4.5.6-SNAPSHOT", - hash: "curagt", - versionedHome: filepath.Join("data", "elastic-agent-4.5.6-SNAPSHOT-curagt"), - }, - previousAgent: agentInstall{ - parsedVersion: parsed123SNAPSHOT, - version: "1.2.3-SNAPSHOT", - hash: "prvagt", - versionedHome: filepath.Join("data", "elastic-agent-1.2.3-SNAPSHOT-prvagt"), - }, - action: nil, - details: details.NewDetails("4.5.6-SNAPSHOT", details.StateReplacing, ""), - rollbackWindow: 7 * 24 * time.Hour, - }, - wantErr: assert.NoError, - assertAfterMark: func(t *testing.T, dataDir string) { - actualMarker, err := LoadMarker(dataDir) - require.NoError(t, err, "error reading actualMarker content after writing") - - expectedMarker := &UpdateMarker{ - Version: "4.5.6-SNAPSHOT", - Hash: "curagt", - VersionedHome: filepath.Join("data", "elastic-agent-4.5.6-SNAPSHOT-curagt"), - UpdatedOn: updatedOnNow, - PrevVersion: "1.2.3-SNAPSHOT", - PrevHash: "prvagt", - PrevVersionedHome: filepath.Join("data", "elastic-agent-1.2.3-SNAPSHOT-prvagt"), - Acked: false, - Action: nil, - Details: &details.Details{ - TargetVersion: "4.5.6-SNAPSHOT", - State: "UPG_REPLACING", - ActionID: "", - }, - } - assert.Equal(t, expectedMarker, actualMarker) - }, - }, - { - name: "rollback window specified and new version is at least 9.2.0-SNAPSHOT - available rollbacks must be present", + name: "available rollbacks passed in - available rollbacks must be present in upgrade marker", args: args{ updatedOn: updatedOnNow, currentAgent: agentInstall{ @@ -241,9 +198,14 @@ func TestMarkUpgrade(t *testing.T) { hash: "prvagt", versionedHome: filepath.Join("data", "elastic-agent-1.2.3-SNAPSHOT-prvagt"), }, - action: nil, - details: details.NewDetails("9.2.0-SNAPSHOT", details.StateReplacing, ""), - rollbackWindow: 7 * 24 * time.Hour, + action: nil, + details: details.NewDetails("9.2.0-SNAPSHOT", details.StateReplacing, ""), + availableRollbacks: map[string]TTLMarker{ + filepath.Join("data", "elastic-agent-1.2.3-SNAPSHOT-prvagt"): { + Version: "1.2.3-SNAPSHOT", + ValidUntil: twentyFourHoursFromNow, + }, + }, }, wantErr: assert.NoError, assertAfterMark: func(t *testing.T, dataDir string) { @@ -266,11 +228,10 @@ func TestMarkUpgrade(t *testing.T) { ActionID: "", Metadata: details.Metadata{}, }, - RollbacksAvailable: []RollbackAvailable{ - { + RollbacksAvailable: map[string]TTLMarker{ + filepath.Join("data", "elastic-agent-1.2.3-SNAPSHOT-prvagt"): { Version: "1.2.3-SNAPSHOT", - Home: filepath.Join("data", "elastic-agent-1.2.3-SNAPSHOT-prvagt"), - ValidUntil: updatedOnNow.Add(7 * 24 * time.Hour), + ValidUntil: twentyFourHoursFromNow, }, }, } @@ -295,7 +256,7 @@ func TestMarkUpgrade(t *testing.T) { tc.setupBeforeMark(t, dataDir) } - err := markUpgrade(log, dataDir, tc.args.updatedOn, tc.args.currentAgent, tc.args.previousAgent, tc.args.action, tc.args.details, tc.args.rollbackWindow) + err := markUpgrade(log, dataDir, tc.args.updatedOn, tc.args.currentAgent, tc.args.previousAgent, tc.args.action, tc.args.details, tc.args.availableRollbacks) tc.wantErr(t, err) if tc.assertAfterMark != nil { tc.assertAfterMark(t, dataDir) diff --git a/internal/pkg/agent/application/upgrade/ttl_marker_source.go b/internal/pkg/agent/application/upgrade/ttl_marker_source.go new file mode 100644 index 00000000000..4445ff4526e --- /dev/null +++ b/internal/pkg/agent/application/upgrade/ttl_marker_source.go @@ -0,0 +1,125 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License 2.0; +// you may not use this file except in compliance with the Elastic License 2.0. + +package upgrade + +import ( + "fmt" + "os" + "path/filepath" + + "gopkg.in/yaml.v2" + + "github.com/elastic/elastic-agent/pkg/core/logger" +) + +const ttlMarkerName = ".ttl" + +var defaultMarkerGlobPattern = filepath.Join("data", "elastic-agent-*", ttlMarkerName) + +type TTLMarkerRegistry struct { + baseDir string + markerFileGlobPattern string + log *logger.Logger +} + +func NewTTLMarkerRegistry(log *logger.Logger, baseDir string) *TTLMarkerRegistry { + return &TTLMarkerRegistry{ + baseDir: baseDir, + markerFileGlobPattern: defaultMarkerGlobPattern, + log: log, + } +} + +func (T TTLMarkerRegistry) Set(m map[string]TTLMarker) error { + // identify the marker files to be deleted first + existingMarkers, err := T.Get() + if err != nil { + return fmt.Errorf("accessing existing markers: %w", err) + } + + for versionedHome := range existingMarkers { + _, ok := m[versionedHome] + if !ok { + // the existing marker should not be in the final state + T.log.Debugf("Removing marker for versionedHome: %s", versionedHome) + err = os.Remove(filepath.Join(T.baseDir, versionedHome, ttlMarkerName)) + if err != nil { + return fmt.Errorf("removing ttl marker for %q: %w", versionedHome, err) + } + } + } + + // create all the remaining markers + return T.addOrReplace(m) +} + +func (T TTLMarkerRegistry) Get() (map[string]TTLMarker, error) { + matches, err := filepath.Glob(filepath.Join(T.baseDir, T.markerFileGlobPattern)) + if err != nil { + return nil, fmt.Errorf("failed to glob files using %q: %w", T.markerFileGlobPattern, err) + } + T.log.Debugf("Found matching versionedHomes: %v", matches) + ttlMarkers := make(map[string]TTLMarker, len(matches)) + for _, match := range matches { + T.log.Debugf("Reading marker from versionedHome: %s", match) + relPath, err := filepath.Rel(T.baseDir, filepath.Dir(match)) + if err != nil { + return nil, fmt.Errorf("failed to determine path for %q relative to %q : %w", match, T.baseDir, err) + } + marker, err := readTTLMarker(match) + if err != nil { + return nil, fmt.Errorf("failed to read marker from file %q: %w", match, err) + } + ttlMarkers[relPath] = marker + } + + return ttlMarkers, nil +} + +func (T TTLMarkerRegistry) addOrReplace(m map[string]TTLMarker) error { + for versionedHome, marker := range m { + dstFilePath := filepath.Join(T.baseDir, versionedHome, ttlMarkerName) + err := writeTTLMarker(dstFilePath, marker) + if err != nil { + return fmt.Errorf("writing marker %q: %w", dstFilePath, err) + } + } + + return nil +} + +func readTTLMarker(filePath string) (TTLMarker, error) { + file, err := os.Open(filePath) + if err != nil { + return TTLMarker{}, fmt.Errorf("failed to open %q: %w", filePath, err) + } + defer func(file *os.File) { + _ = file.Close() + }(file) + ttlMarker := TTLMarker{} + err = yaml.NewDecoder(file).Decode(&ttlMarker) + if err != nil { + return TTLMarker{}, fmt.Errorf("failed to decode %q: %w", filePath, err) + } + + return ttlMarker, nil +} + +func writeTTLMarker(filePath string, marker TTLMarker) error { + file, err := os.Create(filePath) + if err != nil { + return fmt.Errorf("failed to open %q: %w", filePath, err) + } + defer func(file *os.File) { + _ = file.Close() + }(file) + + err = yaml.NewEncoder(file).Encode(marker) + if err != nil { + return fmt.Errorf("failed to encode %q: %w", filePath, err) + } + + return nil +} diff --git a/internal/pkg/agent/application/upgrade/ttl_marker_source_test.go b/internal/pkg/agent/application/upgrade/ttl_marker_source_test.go new file mode 100644 index 00000000000..3ac95e1769d --- /dev/null +++ b/internal/pkg/agent/application/upgrade/ttl_marker_source_test.go @@ -0,0 +1,241 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License 2.0; +// you may not use this file except in compliance with the Elastic License 2.0. + +package upgrade + +import ( + "bytes" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + "text/template" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/elastic/elastic-agent/pkg/core/logger/loggertest" +) + +func TestTTLMarkerRegistry_Get(t *testing.T) { + const TTLMarkerYAMLTemplate = ` + version: {{ .Version }} + valid_until: {{ .ValidUntil }}` + + parsedTemplate, err := template.New("ttlMarker").Parse(TTLMarkerYAMLTemplate) + require.NoError(t, err, "error parsing ttl marker template") + + now := time.Now() + nowString := now.Format(time.RFC3339) + // re-parse now to account for loss of fidelity due to marshal/unmarshal + now, _ = time.Parse(time.RFC3339, nowString) + + yesterday := now.Add(-24 * time.Hour) + yesterdayString := yesterday.Format(time.RFC3339) + + tomorrow := now.Add(24 * time.Hour) + tomorrowString := tomorrow.Format(time.RFC3339) + + versions := []string{"1.2.3", "4.5.6", "7.8.9-SNAPSHOT"} + versionedHomes := []string{"elastic-agent-1.2.3-past", "elastic-agent-4.5.6-present", "elastic-agent-7.8.9-SNAPSHOT-future"} + ttls := []string{yesterdayString, nowString, tomorrowString} + + tests := []struct { + name string + setup func(t *testing.T, tmpDir string) + want map[string]TTLMarker + wantErr assert.ErrorAssertionFunc + }{ + { + name: "Empty directory - empty map", + setup: func(t *testing.T, tmpDir string) { + // nothing to do here + }, + want: map[string]TTLMarker{}, + wantErr: assert.NoError, + }, + { + name: "multiple directories, no marker - empty map", + setup: func(t *testing.T, tmpDir string) { + for _, versionedHome := range versionedHomes { + err := os.MkdirAll(filepath.Join(tmpDir, "data", versionedHome), 0755) + require.NoError(t, err, "error setting up fake agent install directory") + } + }, + want: map[string]TTLMarker{}, + wantErr: assert.NoError, + }, + { + name: "multiple directories, ttl on past and present marker - return value", + setup: func(t *testing.T, tmpDir string) { + + for i, versionedHome := range versionedHomes { + err := os.MkdirAll(filepath.Join(tmpDir, "data", versionedHome), 0755) + require.NoError(t, err, "error setting up fake agent install directory") + + if i < 2 { + buf := bytes.Buffer{} + err = parsedTemplate.Execute(&buf, map[string]string{"Version": versions[i], "ValidUntil": ttls[i]}) + require.NoError(t, err, "error executing ttl marker template") + err = os.WriteFile(filepath.Join(tmpDir, "data", versionedHome, ttlMarkerName), buf.Bytes(), 0644) + require.NoError(t, err, "error setting up fake agent ttl marker") + } + } + }, + want: map[string]TTLMarker{ + filepath.Join("data", "elastic-agent-1.2.3-past"): { + Version: "1.2.3", + ValidUntil: yesterday, + }, + filepath.Join("data", "elastic-agent-4.5.6-present"): { + Version: "4.5.6", + ValidUntil: now, + }, + }, + wantErr: assert.NoError, + }, + { + name: "empty marker - error", + setup: func(t *testing.T, tmpDir string) { + for _, versionedHome := range versionedHomes { + err := os.MkdirAll(filepath.Join(tmpDir, "data", versionedHome), 0755) + require.NoError(t, err, "error setting up fake agent install directory") + err = os.WriteFile(filepath.Join(tmpDir, "data", versionedHome, ttlMarkerName), nil, 0644) + require.NoError(t, err, "error setting up fake agent ttl marker") + } + }, + want: nil, + wantErr: assert.Error, + }, + { + name: "ttl content is not yaml - error", + setup: func(t *testing.T, tmpDir string) { + err := os.MkdirAll(filepath.Join(tmpDir, "data", versionedHomes[0]), 0755) + require.NoError(t, err, "error setting up fake agent install directory") + err = os.WriteFile(filepath.Join(tmpDir, "data", versionedHomes[0], ttlMarkerName), []byte("this is not yaml"), 0644) + require.NoError(t, err, "error setting up fake agent ttl marker") + }, + want: nil, + wantErr: assert.Error, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tmpDir := t.TempDir() + tt.setup(t, tmpDir) + testLogger, _ := loggertest.New(t.Name()) + T := NewTTLMarkerRegistry(testLogger, tmpDir) + got, err := T.Get() + if !tt.wantErr(t, err, "Get()") { + return + } + assert.Equal(t, tt.want, got, "Get()") + }) + } +} + +func TestTTLMarkerRegistry_Set(t *testing.T) { + const TTLMarkerYAMLTemplate = ` + version: {{ .Version }} + valid_until: {{ .ValidUntil }}` + + expectedMarkerContentTemplate, err := template.New("expected marker").Parse(TTLMarkerYAMLTemplate) + require.NoError(t, err) + + now := time.Now() + nowString := now.Format(time.RFC3339) + // re-parse now to account for loss of fidelity due to marshal/unmarshal + now, _ = time.Parse(time.RFC3339, nowString) + + tomorrow := now.Add(24 * time.Hour) + tomorrowString := tomorrow.Format(time.RFC3339) + tomorrow, _ = time.Parse(time.RFC3339, tomorrowString) + + versions := []string{"1.2.3", "4.5.6"} + versionedHomes := []string{"elastic-agent-1.2.3-past", "elastic-agent-4.5.6-present"} + ttls := []string{tomorrowString, ""} + + type args struct { + m map[string]TTLMarker + } + tests := []struct { + name string + setup func(t *testing.T, tmpDir string) + args args + wantErr assert.ErrorAssertionFunc + postAssertions func(t *testing.T, tmpDir string) + }{ + { + name: "no ttl are present - all get created", + setup: func(t *testing.T, tmpDir string) { + for _, versionedHome := range versionedHomes { + err := os.MkdirAll(filepath.Join(tmpDir, "data", versionedHome), 0755) + require.NoError(t, err, "error setting up fake agent install directory") + } + }, + args: args{ + map[string]TTLMarker{ + filepath.Join("data", versionedHomes[0]): { + Version: versions[0], + ValidUntil: tomorrow, + }, + }, + }, + wantErr: assert.NoError, + postAssertions: func(t *testing.T, tmpDir string) { + notExistingTTLMarkerFilePath := filepath.Join(tmpDir, "data", versionedHomes[1], ttlMarkerName) + assert.NoFileExists(t, notExistingTTLMarkerFilePath) + expectedTTLMarkerFilePath := filepath.Join(tmpDir, "data", versionedHomes[0], ttlMarkerName) + if assert.FileExists(t, expectedTTLMarkerFilePath, "new TTL marker should have been created") { + + b := new(strings.Builder) + err = expectedMarkerContentTemplate.Execute(b, map[string]string{"Version": versions[0], "ValidUntil": ttls[0]}) + require.NoError(t, err) + actualMarkerContent, err := os.ReadFile(expectedTTLMarkerFilePath) + require.NoError(t, err) + assert.YAMLEq(t, b.String(), string(actualMarkerContent)) + } + }, + }, + { + name: "ttls are present, none are specified - all deleted", + setup: func(t *testing.T, tmpDir string) { + for i, versionedHome := range versionedHomes { + err = os.MkdirAll(filepath.Join(tmpDir, "data", versionedHome), 0755) + require.NoError(t, err, "error setting up fake agent install directory") + b := new(strings.Builder) + err = expectedMarkerContentTemplate.Execute(b, map[string]string{"Version": versions[i], "ValidUntil": ttls[i]}) + require.NoError(t, err, "error setting up ttl marker") + err = os.WriteFile(filepath.Join(tmpDir, "data", versionedHomes[i], ttlMarkerName), []byte(b.String()), 0644) + } + }, + args: args{ + nil, + }, + wantErr: assert.NoError, + postAssertions: func(t *testing.T, tmpDir string) { + for _, versionedHome := range versionedHomes { + notExistingTTLMarkerFilePath := filepath.Join(tmpDir, "data", versionedHome, ttlMarkerName) + assert.NoFileExists(t, notExistingTTLMarkerFilePath) + } + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tmpDir := t.TempDir() + if tt.setup != nil { + tt.setup(t, tmpDir) + } + testLogger, _ := loggertest.New(t.Name()) + T := NewTTLMarkerRegistry(testLogger, tmpDir) + tt.wantErr(t, T.Set(tt.args.m), fmt.Sprintf("Set(%v)", tt.args.m)) + if tt.postAssertions != nil { + tt.postAssertions(t, tmpDir) + } + }) + } +} diff --git a/internal/pkg/agent/application/upgrade/upgrade.go b/internal/pkg/agent/application/upgrade/upgrade.go index fbc2b38f414..359fdf25fa5 100644 --- a/internal/pkg/agent/application/upgrade/upgrade.go +++ b/internal/pkg/agent/application/upgrade/upgrade.go @@ -16,7 +16,7 @@ import ( "strings" "time" - "github.com/otiai10/copy" + filecopy "github.com/otiai10/copy" "go.elastic.co/apm/v2" "github.com/elastic/elastic-agent/internal/pkg/agent/application/filelock" @@ -66,9 +66,9 @@ var ( ErrNilUpdateMarker = errors.New("loaded a nil update marker") ErrEmptyRollbackVersion = errors.New("rollback version is empty") ErrNoRollbacksAvailable = errors.New("no rollbacks available") - - // Version_9_2_0_SNAPSHOT is the minimum version for manual rollback and rollback reason - Version_9_2_0_SNAPSHOT = agtversion.NewParsedSemVer(9, 2, 0, "SNAPSHOT", "") + ErrAgentInstallNotFound = errors.New("agent install descriptor not found") + // Version_9_3_0_SNAPSHOT is the minimum version for manual rollback and rollback reason + Version_9_3_0_SNAPSHOT = agtversion.NewParsedSemVer(9, 3, 0, "SNAPSHOT", "") ) func init() { @@ -89,10 +89,10 @@ type unpackHandler interface { // Types used to abstract copyActionStore, copyRunDirectory and github.com/otiai10/copy.Copy type copyActionStoreFunc func(log *logger.Logger, newHome string) error type copyRunDirectoryFunc func(log *logger.Logger, oldRunPath, newRunPath string) error -type fileDirCopyFunc func(from, to string, opts ...copy.Options) error -type markUpgradeFunc func(log *logger.Logger, dataDirPath string, updatedOn time.Time, agent, previousAgent agentInstall, action *fleetapi.ActionUpgrade, upgradeDetails *details.Details, rollbackWindow time.Duration) error +type fileDirCopyFunc func(from, to string, opts ...filecopy.Options) error +type markUpgradeFunc func(log *logger.Logger, dataDirPath string, updatedOn time.Time, agent, previousAgent agentInstall, action *fleetapi.ActionUpgrade, upgradeDetails *details.Details, availableRollbacks map[string]TTLMarker) error type changeSymlinkFunc func(log *logger.Logger, topDirPath, symlinkPath, newTarget string) error -type rollbackInstallFunc func(ctx context.Context, log *logger.Logger, topDirPath, versionedHome, oldVersionedHome string) error +type rollbackInstallFunc func(ctx context.Context, log *logger.Logger, topDirPath, versionedHome, oldVersionedHome string, rollbackSource availableRollbacksSource) error // Types used to abstract stdlib functions type mkdirAllFunc func(name string, perm fs.FileMode) error @@ -116,16 +116,22 @@ type WatcherHelper interface { TakeOverWatcher(ctx context.Context, log *logger.Logger, topDir string) (*filelock.AppLocker, error) } +type availableRollbacksSource interface { + Set(map[string]TTLMarker) error + Get() (map[string]TTLMarker, error) +} + // Upgrader performs an upgrade type Upgrader struct { - log *logger.Logger - settings *artifact.Config - upgradeSettings *configuration.UpgradeConfig - agentInfo info.Agent - upgradeable bool - fleetServerURI string - markerWatcher MarkerWatcher - watcherHelper WatcherHelper + log *logger.Logger + settings *artifact.Config + upgradeSettings *configuration.UpgradeConfig + agentInfo info.Agent + upgradeable bool + fleetServerURI string + markerWatcher MarkerWatcher + watcherHelper WatcherHelper + availableRollbacksSource availableRollbacksSource // The following are abstractions for testability artifactDownloader artifactDownloadHandler @@ -147,24 +153,25 @@ func IsUpgradeable() bool { } // NewUpgrader creates an upgrader which is capable of performing upgrade operation -func NewUpgrader(log *logger.Logger, settings *artifact.Config, upgradeConfig *configuration.UpgradeConfig, agentInfo info.Agent, watcherHelper WatcherHelper) (*Upgrader, error) { +func NewUpgrader(log *logger.Logger, settings *artifact.Config, upgradeConfig *configuration.UpgradeConfig, agentInfo info.Agent, watcherHelper WatcherHelper, ars availableRollbacksSource) (*Upgrader, error) { return &Upgrader{ - log: log, - settings: settings, - upgradeSettings: upgradeConfig, - agentInfo: agentInfo, - upgradeable: IsUpgradeable(), - markerWatcher: newMarkerFileWatcher(markerFilePath(paths.Data()), log), - watcherHelper: watcherHelper, - artifactDownloader: newArtifactDownloader(settings, log), - unpacker: newUnpacker(log), - isDiskSpaceErrorFunc: upgradeErrors.IsDiskSpaceError, - extractAgentVersion: extractAgentVersion, - copyActionStore: copyActionStoreProvider(os.ReadFile, os.WriteFile), - copyRunDirectory: copyRunDirectoryProvider(os.MkdirAll, copy.Copy), - markUpgrade: markUpgradeProvider(UpdateActiveCommit, os.WriteFile), - changeSymlink: changeSymlink, - rollbackInstall: rollbackInstall, + log: log, + settings: settings, + upgradeSettings: upgradeConfig, + agentInfo: agentInfo, + upgradeable: IsUpgradeable(), + markerWatcher: newMarkerFileWatcher(markerFilePath(paths.Data()), log), + watcherHelper: watcherHelper, + availableRollbacksSource: ars, + artifactDownloader: newArtifactDownloader(settings, log), + unpacker: newUnpacker(log), + isDiskSpaceErrorFunc: upgradeErrors.IsDiskSpaceError, + extractAgentVersion: extractAgentVersion, + copyActionStore: copyActionStoreProvider(os.ReadFile, os.WriteFile), + copyRunDirectory: copyRunDirectoryProvider(os.MkdirAll, filecopy.Copy), + markUpgrade: markUpgradeProvider(UpdateActiveCommit, os.WriteFile), + changeSymlink: changeSymlink, + rollbackInstall: rollbackInstall, }, nil } @@ -433,10 +440,15 @@ func (u *Upgrader) Upgrade(ctx context.Context, version string, rollback bool, s if err := u.changeSymlink(u.log, paths.Top(), symlinkPath, newPath); err != nil { u.log.Errorw("Rolling back: changing symlink failed", "error.message", err) - rollbackErr := u.rollbackInstall(ctx, u.log, paths.Top(), hashedDir, currentVersionedHome) + rollbackErr := u.rollbackInstall(ctx, u.log, paths.Top(), hashedDir, currentVersionedHome, u.availableRollbacksSource) return nil, goerrors.Join(err, rollbackErr) } + rollbackWindow := disableRollbackWindow + if u.upgradeSettings != nil && u.upgradeSettings.Rollback != nil { + rollbackWindow = u.upgradeSettings.Rollback.Window + } + // We rotated the symlink successfully: prepare the current and previous agent installation details for the update marker // In update marker the `current` agent install is the one where the symlink is pointing (the new one we didn't start yet) // while the `previous` install is the currently executing elastic-agent that is no longer reachable via the symlink. @@ -455,18 +467,23 @@ func (u *Upgrader) Upgrade(ctx context.Context, version string, rollback bool, s hash: release.Commit(), versionedHome: currentVersionedHome, } - rollbackWindow := time.Duration(0) - if u.upgradeSettings != nil && u.upgradeSettings.Rollback != nil { - rollbackWindow = u.upgradeSettings.Rollback.Window + + availableRollbacks := getAvailableRollbacks(rollbackWindow, time.Now(), release.VersionWithSnapshot(), previousParsedVersion, currentVersionedHome) + + if err = u.availableRollbacksSource.Set(availableRollbacks); err != nil { + u.log.Errorw("Rolling back: setting ttl markers failed", "error.message", err) + rollbackErr := u.rollbackInstall(ctx, u.log, paths.Top(), hashedDir, currentVersionedHome, u.availableRollbacksSource) + return nil, goerrors.Join(err, rollbackErr) } - if err := u.markUpgrade(u.log, + + if err = u.markUpgrade(u.log, paths.Data(), // data dir to place the marker in time.Now(), current, // new agent version data previous, // old agent version data - action, det, rollbackWindow); err != nil { + action, det, availableRollbacks); err != nil { u.log.Errorw("Rolling back: marking upgrade failed", "error.message", err) - rollbackErr := u.rollbackInstall(ctx, u.log, paths.Top(), hashedDir, currentVersionedHome) + rollbackErr := u.rollbackInstall(ctx, u.log, paths.Top(), hashedDir, currentVersionedHome, u.availableRollbacksSource) return nil, goerrors.Join(err, rollbackErr) } @@ -475,14 +492,14 @@ func (u *Upgrader) Upgrade(ctx context.Context, version string, rollback bool, s var watcherCmd *exec.Cmd if watcherCmd, err = u.watcherHelper.InvokeWatcher(u.log, watcherExecutable); err != nil { u.log.Errorw("Rolling back: starting watcher failed", "error.message", err) - rollbackErr := u.rollbackInstall(ctx, u.log, paths.Top(), hashedDir, currentVersionedHome) + rollbackErr := u.rollbackInstall(ctx, u.log, paths.Top(), hashedDir, currentVersionedHome, u.availableRollbacksSource) return nil, goerrors.Join(err, rollbackErr) } watcherWaitErr := u.watcherHelper.WaitForWatcher(ctx, u.log, markerFilePath(paths.Data()), watcherMaxWaitTime) if watcherWaitErr != nil { killWatcherErr := watcherCmd.Process.Kill() - rollbackErr := u.rollbackInstall(ctx, u.log, paths.Top(), hashedDir, currentVersionedHome) + rollbackErr := u.rollbackInstall(ctx, u.log, paths.Top(), hashedDir, currentVersionedHome, u.availableRollbacksSource) return nil, goerrors.Join(watcherWaitErr, killWatcherErr, rollbackErr) } @@ -498,6 +515,28 @@ func (u *Upgrader) Upgrade(ctx context.Context, version string, rollback bool, s return cb, nil } +func getAvailableRollbacks(rollbackWindow time.Duration, now time.Time, currentVersion string, parsedCurrentVersion *agtversion.ParsedSemVer, currentVersionedHome string) map[string]TTLMarker { + if rollbackWindow == 0 { + // if there's no rollback window it means that no rollback should survive the watcher cleanup at the end of the grace period. + return nil + } + + if parsedCurrentVersion == nil || parsedCurrentVersion.Less(*Version_9_3_0_SNAPSHOT) { + // the version we are upgrading to does not support manual rollbacks + return nil + } + + // when multiple rollbacks will be supported, read the existing descriptor + // at this stage we can get by with a single rollback + res := make(map[string]TTLMarker, 1) + res[currentVersionedHome] = TTLMarker{ + Version: currentVersion, + ValidUntil: now.Add(rollbackWindow), + } + + return res +} + func (u *Upgrader) rollbackToPreviousVersion(ctx context.Context, topDir string, now time.Time, version string, action *fleetapi.ActionUpgrade) (reexec.ShutdownCallbackFn, error) { if version == "" { return nil, ErrEmptyRollbackVersion @@ -541,7 +580,7 @@ func (u *Upgrader) rollbackToPreviousVersion(ctx context.Context, topDir string, if len(updateMarker.RollbacksAvailable) == 0 { return ErrNoRollbacksAvailable } - var selectedRollback *RollbackAvailable + var selectedRollback *TTLMarker for _, rollback := range updateMarker.RollbacksAvailable { if rollback.Version == version && now.Before(rollback.ValidUntil) { selectedRollback = &rollback @@ -692,7 +731,7 @@ func isSameVersion(log *logger.Logger, current agentVersion, newVersion agentVer return current == newVersion } -func rollbackInstall(ctx context.Context, log *logger.Logger, topDirPath, versionedHome, oldVersionedHome string) error { +func rollbackInstall(ctx context.Context, log *logger.Logger, topDirPath, versionedHome, oldVersionedHome string, rollbackSource availableRollbacksSource) error { oldAgentPath := paths.BinaryPath(filepath.Join(topDirPath, oldVersionedHome), agentName) err := changeSymlink(log, topDirPath, filepath.Join(topDirPath, agentName), oldAgentPath) if err != nil && !errors.Is(err, fs.ErrNotExist) { @@ -704,6 +743,12 @@ func rollbackInstall(ctx context.Context, log *logger.Logger, topDirPath, versio if err != nil && !errors.Is(err, fs.ErrNotExist) { return fmt.Errorf("rolling back install: removing new agent install at %q failed: %w", newAgentInstallPath, err) } + + err = rollbackSource.Set(nil) + if err != nil { + return fmt.Errorf("rolling back install: error clearing ttl markers: %w", err) + } + return nil } @@ -783,7 +828,7 @@ func shutdownCallback(l *logger.Logger, homePath, prevVersion, newVersion, newHo newRelPath = strings.ReplaceAll(newRelPath, oldHome, newHome) newDir := filepath.Join(newHome, newRelPath) l.Debugf("copying %q -> %q", processDir, newDir) - if err := copyDir(l, processDir, newDir, true, copy.Copy); err != nil { + if err := copyDir(l, processDir, newDir, true, filecopy.Copy); err != nil { return err } } @@ -855,9 +900,9 @@ func copyDir(l *logger.Logger, from, to string, ignoreErrs bool, fileDirCopy fil copyConcurrency = runtime.NumCPU() * 4 } - return fileDirCopy(from, to, copy.Options{ - OnSymlink: func(_ string) copy.SymlinkAction { - return copy.Shallow + return fileDirCopy(from, to, filecopy.Options{ + OnSymlink: func(_ string) filecopy.SymlinkAction { + return filecopy.Shallow }, Sync: true, OnError: onErr, diff --git a/internal/pkg/agent/application/upgrade/upgrade_test.go b/internal/pkg/agent/application/upgrade/upgrade_test.go index 5bc96564cb6..a019fa0e2b5 100644 --- a/internal/pkg/agent/application/upgrade/upgrade_test.go +++ b/internal/pkg/agent/application/upgrade/upgrade_test.go @@ -1044,43 +1044,42 @@ func TestIsSameReleaseVersion(t *testing.T) { func TestManualRollback(t *testing.T) { const updatemarkerwatching456NoRollbackAvailable = ` - version: 4.5.6 - hash: newver - versioned_home: data/elastic-agent-4.5.6-newver - updated_on: 2025-07-11T10:11:12.131415Z - prev_version: 1.2.3 - prev_hash: oldver - prev_versioned_home: data/elastic-agent-1.2.3-oldver - acked: false - action: null - details: - target_version: 4.5.6 - state: UPG_WATCHING - metadata: - retry_until: null - desired_outcome: UPGRADE - ` + version: 4.5.6 + hash: newver + versioned_home: data/elastic-agent-4.5.6-newver + updated_on: 2025-07-11T10:11:12.131415Z + prev_version: 1.2.3 + prev_hash: oldver + prev_versioned_home: data/elastic-agent-1.2.3-oldver + acked: false + action: null + details: + target_version: 4.5.6 + state: UPG_WATCHING + metadata: + retry_until: null + ` const updatemarkerwatching456 = ` - version: 4.5.6 - hash: newver - versioned_home: data/elastic-agent-4.5.6-newver - updated_on: 2025-07-11T10:11:12.131415Z - prev_version: 1.2.3 - prev_hash: oldver - prev_versioned_home: data/elastic-agent-1.2.3-oldver - acked: false - action: null - details: - target_version: 4.5.6 - state: UPG_WATCHING - metadata: - retry_until: null - desired_outcome: UPGRADE - rollbacks_available: - - version: 1.2.3 - home: data/elastic-agent-1.2.3-oldver - valid_until: 2025-07-18T10:11:12.131415Z - ` + version: 4.5.6 + hash: newver + versioned_home: data/elastic-agent-4.5.6-newver + updated_on: 2025-07-11T10:11:12.131415Z + prev_version: 1.2.3 + prev_hash: oldver + prev_versioned_home: data/elastic-agent-1.2.3-oldver + acked: false + action: null + details: + target_version: 4.5.6 + state: UPG_WATCHING + metadata: + retry_until: null + desired_outcome: UPGRADE + rollbacks_available: + "data/elastic-agent-1.2.3-oldver": + version: 1.2.3 + valid_until: 2025-07-18T10:11:12.131415Z + ` parsed123Version, err := agtversion.ParseVersion("1.2.3") require.NoError(t, err) @@ -1292,6 +1291,7 @@ func TestManualRollback(t *testing.T) { log, _ := loggertest.New(t.Name()) mockAgentInfo := info.NewMockAgent(t) mockWatcherHelper := NewMockWatcherHelper(t) + mockRollbacksSource := newMockAvailableRollbacksSource(t) topDir := t.TempDir() err := os.MkdirAll(paths.DataFrom(topDir), 0777) require.NoError(t, err, "error creating data directory in topDir %q", topDir) @@ -1300,7 +1300,7 @@ func TestManualRollback(t *testing.T) { tc.setup(t, topDir, mockAgentInfo, mockWatcherHelper) } - upgrader, err := NewUpgrader(log, tc.artifactSettings, tc.upgradeSettings, mockAgentInfo, mockWatcherHelper) + upgrader, err := NewUpgrader(log, tc.artifactSettings, tc.upgradeSettings, mockAgentInfo, mockWatcherHelper, mockRollbacksSource) require.NoError(t, err, "error instantiating upgrader") _, err = upgrader.rollbackToPreviousVersion(t.Context(), topDir, tc.now, tc.version, nil) tc.wantErr(t, err, "unexpected error returned by rollbackToPreviousVersion()") @@ -1352,6 +1352,7 @@ func TestUpgradeErrorHandling(t *testing.T) { upgraderMocker upgraderMocker checkArchiveCleanup bool checkVersionedHomeCleanup bool + setupMocks func(t *testing.T, mockAgentInfo *info.MockAgent, mockRollbackSrc *mockAvailableRollbacksSource, mockWatcherHelper *MockWatcherHelper) } testCases := map[string]testCase{ @@ -1365,6 +1366,9 @@ func TestUpgradeErrorHandling(t *testing.T) { } }, checkArchiveCleanup: true, + setupMocks: func(t *testing.T, mockAgentInfo *info.MockAgent, mockRollbackSrc *mockAvailableRollbacksSource, mockWatcherHelper *MockWatcherHelper) { + mockAgentInfo.EXPECT().Version().Return("9.0.0") + }, }, "should return error if getPackageMetadata fails": { isDiskSpaceErrorResult: false, @@ -1378,6 +1382,9 @@ func TestUpgradeErrorHandling(t *testing.T) { } }, checkArchiveCleanup: true, + setupMocks: func(t *testing.T, mockAgentInfo *info.MockAgent, mockRollbackSrc *mockAvailableRollbacksSource, mockWatcherHelper *MockWatcherHelper) { + mockAgentInfo.EXPECT().Version().Return("9.0.0") + }, }, "should return error and cleanup downloaded archive if unpack fails before extracting": { isDiskSpaceErrorResult: false, @@ -1402,6 +1409,9 @@ func TestUpgradeErrorHandling(t *testing.T) { } }, checkArchiveCleanup: true, + setupMocks: func(t *testing.T, mockAgentInfo *info.MockAgent, mockRollbackSrc *mockAvailableRollbacksSource, mockWatcherHelper *MockWatcherHelper) { + mockAgentInfo.EXPECT().Version().Return("9.0.0") + }, }, "should return error and cleanup downloaded archive if unpack fails after extracting": { isDiskSpaceErrorResult: false, @@ -1431,6 +1441,9 @@ func TestUpgradeErrorHandling(t *testing.T) { }, checkArchiveCleanup: true, checkVersionedHomeCleanup: true, + setupMocks: func(t *testing.T, mockAgentInfo *info.MockAgent, mockRollbackSrc *mockAvailableRollbacksSource, mockWatcherHelper *MockWatcherHelper) { + mockAgentInfo.EXPECT().Version().Return("9.0.0") + }, }, "should return error and cleanup downloaded artifact and extracted archive if copyActionStore fails": { isDiskSpaceErrorResult: false, @@ -1462,6 +1475,9 @@ func TestUpgradeErrorHandling(t *testing.T) { }, checkArchiveCleanup: true, checkVersionedHomeCleanup: true, + setupMocks: func(t *testing.T, mockAgentInfo *info.MockAgent, mockRollbackSrc *mockAvailableRollbacksSource, mockWatcherHelper *MockWatcherHelper) { + mockAgentInfo.EXPECT().Version().Return("9.0.0") + }, }, "should return error and cleanup downloaded artifact and extracted archive if copyRunDirectory fails": { isDiskSpaceErrorResult: false, @@ -1497,6 +1513,9 @@ func TestUpgradeErrorHandling(t *testing.T) { }, checkArchiveCleanup: true, checkVersionedHomeCleanup: true, + setupMocks: func(t *testing.T, mockAgentInfo *info.MockAgent, mockRollbackSrc *mockAvailableRollbacksSource, mockWatcherHelper *MockWatcherHelper) { + mockAgentInfo.EXPECT().Version().Return("9.0.0") + }, }, "should return error and cleanup downloaded artifact and extracted archive if changeSymlink fails": { isDiskSpaceErrorResult: false, @@ -1528,7 +1547,7 @@ func TestUpgradeErrorHandling(t *testing.T) { upgrader.copyRunDirectory = func(log *logger.Logger, oldRunPath, newRunPath string) error { return nil } - upgrader.rollbackInstall = func(ctx context.Context, log *logger.Logger, topDirPath, versionedHome, oldVersionedHome string) error { + upgrader.rollbackInstall = func(ctx context.Context, log *logger.Logger, topDirPath, versionedHome, oldVersionedHome string, source availableRollbacksSource) error { return nil } upgrader.changeSymlink = func(log *logger.Logger, topDirPath, symlinkPath, newTarget string) error { @@ -1537,6 +1556,9 @@ func TestUpgradeErrorHandling(t *testing.T) { }, checkArchiveCleanup: true, checkVersionedHomeCleanup: true, + setupMocks: func(t *testing.T, mockAgentInfo *info.MockAgent, mockRollbackSrc *mockAvailableRollbacksSource, mockWatcherHelper *MockWatcherHelper) { + mockAgentInfo.EXPECT().Version().Return("9.0.0") + }, }, "should return error and cleanup downloaded artifact and extracted archive if markUpgrade fails": { isDiskSpaceErrorResult: false, @@ -1571,15 +1593,19 @@ func TestUpgradeErrorHandling(t *testing.T) { upgrader.changeSymlink = func(log *logger.Logger, topDirPath, symlinkPath, newTarget string) error { return nil } - upgrader.rollbackInstall = func(ctx context.Context, log *logger.Logger, topDirPath, versionedHome, oldVersionedHome string) error { + upgrader.rollbackInstall = func(ctx context.Context, log *logger.Logger, topDirPath, versionedHome, oldVersionedHome string, source availableRollbacksSource) error { return nil } - upgrader.markUpgrade = func(log *logger.Logger, dataDirPath string, updatedOn time.Time, agent, previousAgent agentInstall, action *fleetapi.ActionUpgrade, upgradeDetails *details.Details, rollbackWindow time.Duration) error { + upgrader.markUpgrade = func(log *logger.Logger, dataDirPath string, updatedOn time.Time, agent, previousAgent agentInstall, action *fleetapi.ActionUpgrade, upgradeDetails *details.Details, availableRollbacks map[string]TTLMarker) error { return testError } }, checkArchiveCleanup: true, checkVersionedHomeCleanup: true, + setupMocks: func(t *testing.T, mockAgentInfo *info.MockAgent, mockRollbackSrc *mockAvailableRollbacksSource, mockWatcherHelper *MockWatcherHelper) { + mockAgentInfo.EXPECT().Version().Return("9.0.0") + mockRollbackSrc.EXPECT().Set(map[string]TTLMarker(nil)).Return(nil) + }, }, "should add disk space error to the error chain if downloadArtifact fails with disk space error": { isDiskSpaceErrorResult: true, @@ -1589,19 +1615,29 @@ func TestUpgradeErrorHandling(t *testing.T) { returnError: testError, } }, + setupMocks: func(t *testing.T, mockAgentInfo *info.MockAgent, mockRollbackSrc *mockAvailableRollbacksSource, mockWatcherHelper *MockWatcherHelper) { + mockAgentInfo.EXPECT().Version().Return("9.0.0") + }, }, } - mockAgentInfo := info.NewMockAgent(t) - mockAgentInfo.On("Version").Return("9.0.0") - for name, tc := range testCases { t.Run(name, func(t *testing.T) { baseDir := t.TempDir() paths.SetTop(baseDir) + mockAgentInfo := info.NewMockAgent(t) + mockRollbackSource := newMockAvailableRollbacksSource(t) mockWatcherHelper := NewMockWatcherHelper(t) - upgrader, err := NewUpgrader(log, &artifact.Config{}, nil, mockAgentInfo, mockWatcherHelper) + + if tc.setupMocks != nil { + // setup mocks + tc.setupMocks(t, mockAgentInfo, mockRollbackSource, mockWatcherHelper) + } else { + t.Log("skipping mocks setup as the testcase does not define a setupMocks()") + } + + upgrader, err := NewUpgrader(log, &artifact.Config{}, nil, mockAgentInfo, mockWatcherHelper, mockRollbackSource) require.NoError(t, err) tc.upgraderMocker(upgrader, filepath.Join(baseDir, "mockArchive"), "versionedHome") diff --git a/internal/pkg/agent/cmd/run.go b/internal/pkg/agent/cmd/run.go index bc386c7263e..077d948d318 100644 --- a/internal/pkg/agent/cmd/run.go +++ b/internal/pkg/agent/cmd/run.go @@ -31,7 +31,6 @@ import ( "github.com/elastic/elastic-agent-libs/service" "github.com/elastic/elastic-agent-system-metrics/report" "github.com/elastic/elastic-agent/internal/pkg/agent/vault" - "github.com/elastic/elastic-agent/internal/pkg/diagnostics" "github.com/elastic/elastic-agent/internal/pkg/agent/application" "github.com/elastic/elastic-agent/internal/pkg/agent/application/coordinator" @@ -43,7 +42,6 @@ import ( "github.com/elastic/elastic-agent/internal/pkg/agent/application/reexec" "github.com/elastic/elastic-agent/internal/pkg/agent/application/secret" "github.com/elastic/elastic-agent/internal/pkg/agent/application/upgrade" - "github.com/elastic/elastic-agent/internal/pkg/agent/application/upgrade/details" "github.com/elastic/elastic-agent/internal/pkg/agent/configuration" "github.com/elastic/elastic-agent/internal/pkg/agent/errors" "github.com/elastic/elastic-agent/internal/pkg/agent/install" @@ -52,6 +50,7 @@ import ( "github.com/elastic/elastic-agent/internal/pkg/cli" "github.com/elastic/elastic-agent/internal/pkg/config" monitoringCfg "github.com/elastic/elastic-agent/internal/pkg/core/monitoring/config" + "github.com/elastic/elastic-agent/internal/pkg/diagnostics" "github.com/elastic/elastic-agent/internal/pkg/release" "github.com/elastic/elastic-agent/pkg/component" "github.com/elastic/elastic-agent/pkg/control/v2/server" @@ -89,6 +88,7 @@ func newRunCommandWithArgs(_ []string, streams *cli.IOStreams) *cobra.Command { testingMode, _ := cmd.Flags().GetBool("testing-mode") if err := run(nil, testingMode, fleetInitTimeout); err != nil && !errors.Is(err, context.Canceled) { fmt.Fprintf(streams.Err, "Error: %v\n%s\n", err, troubleshootMessage) + logExternal(fmt.Sprintf("%s run failed: %s", paths.BinaryName, err)) return err } return nil @@ -166,7 +166,7 @@ func runElasticAgentCritical( var errs []error // early handleUpgrade, but don't error yet - upgradeDetailsFromMarker, err := handleUpgrade() + initialUpdateMarker, err := handleUpgrade() if err != nil { errs = append(errs, fmt.Errorf("failed to handle upgrade: %w", err)) } @@ -232,7 +232,7 @@ func runElasticAgentCritical( } // actually run the agent now - err = runElasticAgent(ctx, cancel, baseLogger, l, cfg, override, stop, testingMode, fleetInitTimeout, upgradeDetailsFromMarker, modifiers...) + err = runElasticAgent(ctx, cancel, baseLogger, l, cfg, override, stop, testingMode, fleetInitTimeout, initialUpdateMarker, modifiers...) return logReturn(l, err) } @@ -247,7 +247,7 @@ func runElasticAgent( stop chan bool, testingMode bool, fleetInitTimeout time.Duration, - upgradeDetailsFromMarker *details.Details, + initialUpgradeMarker *upgrade.UpdateMarker, modifiers ...component.PlatformModifier, ) error { logLvl := logger.DefaultLogLevel @@ -351,7 +351,7 @@ func runElasticAgent( isBootstrap := configuration.IsFleetServerBootstrap(cfg.Fleet) coord, configMgr, _, err := application.New(ctx, l, baseLogger, logLvl, agentInfo, rex, tracer, testingMode, - fleetInitTimeout, isBootstrap, override, upgradeDetailsFromMarker, modifiers...) + fleetInitTimeout, isBootstrap, override, initialUpgradeMarker, modifiers...) if err != nil { return err } @@ -734,7 +734,7 @@ func setupMetrics( // handleUpgrade checks if agent is being run as part of an // ongoing upgrade operation, i.e. being re-exec'd and performs // any upgrade-specific work, if needed. -func handleUpgrade() (*details.Details, error) { +func handleUpgrade() (*upgrade.UpdateMarker, error) { upgradeMarker, err := upgrade.LoadMarker(paths.Data()) if err != nil { return nil, fmt.Errorf("unable to load upgrade marker to check if Agent is being upgraded: %w", err) @@ -753,7 +753,7 @@ func handleUpgrade() (*details.Details, error) { return nil, err } - return upgradeMarker.Details, nil + return upgradeMarker, nil } func ensureInstallMarkerPresent() error { diff --git a/internal/pkg/agent/install/install_test.go b/internal/pkg/agent/install/install_test.go index f2716e493f6..795dcfb0416 100644 --- a/internal/pkg/agent/install/install_test.go +++ b/internal/pkg/agent/install/install_test.go @@ -226,5 +226,6 @@ func TestSetupInstallPath(t *testing.T) { require.NoError(t, err) err = setupInstallPath(tmpdir, ownership) require.NoError(t, err) - require.FileExists(t, filepath.Join(tmpdir, paths.MarkerFileName)) + markerFilePath := filepath.Join(tmpdir, paths.MarkerFileName) + require.FileExists(t, markerFilePath) } diff --git a/pkg/utils/manifest/path_mapper.go b/pkg/utils/manifest/path_mapper.go new file mode 100644 index 00000000000..3d28dd43d4b --- /dev/null +++ b/pkg/utils/manifest/path_mapper.go @@ -0,0 +1,30 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License 2.0; +// you may not use this file except in compliance with the Elastic License 2.0. + +package manifest + +import ( + "path" + "strings" +) + +// PathMapper is a utility object that will help with File mappings specified in a v1/Manifest +type PathMapper struct { + mappings []map[string]string +} + +func (pm PathMapper) Map(packagePath string) string { + for _, mapping := range pm.mappings { + for pkgPath, mappedPath := range mapping { + if strings.HasPrefix(packagePath, pkgPath) { + return path.Join(mappedPath, packagePath[len(pkgPath):]) + } + } + } + return packagePath +} + +func NewPathMapper(mappings []map[string]string) *PathMapper { + return &PathMapper{mappings} +}