From cccee75914be3eeb08f56740aab23891900ad448 Mon Sep 17 00:00:00 2001 From: Arthur Bellal Date: Fri, 10 May 2024 16:21:10 +0200 Subject: [PATCH 01/10] move locks --- cmd/installer/subcommands/daemon/run.go | 2 +- omnibus/config/software/installer.rb | 2 +- omnibus/package-scripts/installer-deb/postrm | 6 +++--- omnibus/package-scripts/installer-rpm/postrm | 6 +++--- pkg/fleet/installer/installer.go | 2 +- .../installer/service/datadog_installer.go | 20 +++++++++++++++---- tasks/installer.py | 3 ++- tasks/libs/common/utils.py | 5 +++++ test/new-e2e/tests/installer/linux_test.go | 4 ++-- 9 files changed, 34 insertions(+), 16 deletions(-) diff --git a/cmd/installer/subcommands/daemon/run.go b/cmd/installer/subcommands/daemon/run.go index 92cb9eb79f4a9..b535f734bc160 100644 --- a/cmd/installer/subcommands/daemon/run.go +++ b/cmd/installer/subcommands/daemon/run.go @@ -65,7 +65,7 @@ func runFxWrapper(global *command.GlobalParams) error { core.Bundle(), fx.Supply(&rcservice.Params{ Options: []service.Option{ - service.WithDatabaseFileName("remote-config-updater.db"), + service.WithDatabaseFileName("remote-config-installer.db"), }, }), rctelemetryreporterimpl.Module(), diff --git a/omnibus/config/software/installer.rb b/omnibus/config/software/installer.rb index 11baa5e85e04d..43f05b30f3afa 100644 --- a/omnibus/config/software/installer.rb +++ b/omnibus/config/software/installer.rb @@ -32,7 +32,7 @@ env = with_embedded_path(env) if linux_target? - command "invoke installer.build --rebuild --install-path=#{install_dir}", env: env + command "invoke installer.build --rebuild --run-path=/var/run/datadog/installer --install-path=#{install_dir}", env: env mkdir "#{install_dir}/bin" mkdir "#{install_dir}/run/" mkdir "#{install_dir}/systemd/" diff --git a/omnibus/package-scripts/installer-deb/postrm b/omnibus/package-scripts/installer-deb/postrm index 4960c54e5ca5a..25fdbc28e12b2 100644 --- a/omnibus/package-scripts/installer-deb/postrm +++ b/omnibus/package-scripts/installer-deb/postrm @@ -6,7 +6,7 @@ PACKAGES_DIR=/opt/datadog-packages INSTALL_DIR=/opt/datadog-installer -PACKAGES_LOCK_DIR=/var/run/datadog-packages +RUN_DIR=/var/run/datadog/installer set -e @@ -14,9 +14,9 @@ case "$1" in remove | purge) datadog-installer purge || true rm -f "/usr/bin/datadog-bootstrap" - echo "Force-deleting $INSTALL_DIR $PACKAGES_DIR $PACKAGES_LOCK_DIR" + echo "Force-deleting $INSTALL_DIR $PACKAGES_DIR $RUN_DIR" rm -rf $PACKAGES_DIR - rm -rf $PACKAGES_LOCK_DIR + rm -rf $RUN_DIR rm -rf $INSTALL_DIR ;; *) diff --git a/omnibus/package-scripts/installer-rpm/postrm b/omnibus/package-scripts/installer-rpm/postrm index 8f876e3e91889..166ccc89ab9a4 100644 --- a/omnibus/package-scripts/installer-rpm/postrm +++ b/omnibus/package-scripts/installer-rpm/postrm @@ -6,7 +6,7 @@ PACKAGES_DIR=/opt/datadog-packages INSTALL_DIR=/opt/datadog-installer -PACKAGES_LOCK_DIR=/var/run/datadog-packages +RUN_DIR=/var/run/datadog/installer set -e @@ -17,9 +17,9 @@ fi datadog-installer purge || true rm -f "/usr/bin/datadog-bootstrap" -echo "Force-deleting $INSTALL_DIR $PACKAGES_DIR $PACKAGES_LOCK_DIR" +echo "Force-deleting $INSTALL_DIR $PACKAGES_DIR $RUN_DIR" rm -rf $PACKAGES_DIR -rm -rf $PACKAGES_LOCK_DIR +rm -rf $RUN_DIR rm -rf $INSTALL_DIR exit 0 diff --git a/pkg/fleet/installer/installer.go b/pkg/fleet/installer/installer.go index a5fdb62a848e1..8f67f1bfec6cd 100644 --- a/pkg/fleet/installer/installer.go +++ b/pkg/fleet/installer/installer.go @@ -28,7 +28,7 @@ const ( // TmpDirPath is the path to the temporary directory used for package installation. TmpDirPath = "/opt/datadog-packages" // LocksPack is the path to the locks directory. - LocksPack = "/var/run/datadog-packages" + LocksPack = "/var/run/datadog/installer/locks" datadogPackageMaxSize = 3 << 30 // 3GiB defaultConfigsDir = "/etc" diff --git a/pkg/fleet/installer/service/datadog_installer.go b/pkg/fleet/installer/service/datadog_installer.go index 6f9773d70158b..ceb2490a83463 100644 --- a/pkg/fleet/installer/service/datadog_installer.go +++ b/pkg/fleet/installer/service/datadog_installer.go @@ -77,19 +77,31 @@ func SetupInstaller(ctx context.Context) (err error) { if err != nil { return fmt.Errorf("error creating /var/log/datadog: %w", err) } - err = os.MkdirAll("/var/run/datadog-packages", 0777) + err = os.MkdirAll("/var/run/datadog/installer", 0755) if err != nil { - return fmt.Errorf("error creating /var/run/datadog-packages: %w", err) + return fmt.Errorf("error creating /var/log/datadog: %w", err) + } + err = os.MkdirAll("/var/run/datadog/installer/locks", 0777) + if err != nil { + return fmt.Errorf("error creating /var/run/datadog/installer/locks: %w", err) } // Locks directory can already be created by a package install - err = os.Chmod("/var/run/datadog-packages", 0777) + err = os.Chmod("/var/run/datadog/installer/locks", 0777) if err != nil { - return fmt.Errorf("error changing permissions of /var/run/datadog-packages: %w", err) + return fmt.Errorf("error changing permissions of /var/run/datadog/installer/locks: %w", err) } err = os.Chown("/var/log/datadog", ddAgentUID, ddAgentGID) if err != nil { return fmt.Errorf("error changing owner of /var/log/datadog: %w", err) } + err = os.Chown("/var/run/datadog", ddAgentUID, ddAgentGID) + if err != nil { + return fmt.Errorf("error changing owner of /var/run/datadog: %w", err) + } + err = os.Chown("/var/run/datadog/installer", ddAgentUID, ddAgentGID) + if err != nil { + return fmt.Errorf("error changing owner of /var/run/datadog: %w", err) + } err = os.Chown("/opt/datadog-packages/datadog-installer/stable/run", ddAgentUID, ddAgentGID) if err != nil { return fmt.Errorf("error changing owner of /opt/datadog-packages/datadog-installer/stable/run: %w", err) diff --git a/tasks/installer.py b/tasks/installer.py index 49a7e1c714692..286973f4a1af7 100644 --- a/tasks/installer.py +++ b/tasks/installer.py @@ -19,6 +19,7 @@ def build( rebuild=False, race=False, install_path=None, + run_path=None, build_include=None, build_exclude=None, arch="x64", @@ -28,7 +29,7 @@ def build( Build the updater. """ - ldflags, gcflags, env = get_build_flags(ctx, major_version=MAJOR_VERSION, install_path=install_path) + ldflags, gcflags, env = get_build_flags(ctx, major_version=MAJOR_VERSION, install_path=install_path, run_path=run_path) build_include = ( get_default_build_tags( diff --git a/tasks/libs/common/utils.py b/tasks/libs/common/utils.py index 3ebd5f1eac7b0..393290b9ccbfb 100644 --- a/tasks/libs/common/utils.py +++ b/tasks/libs/common/utils.py @@ -164,6 +164,7 @@ def get_build_flags( static=False, prefix=None, install_path=None, + run_path=None, embedded_path=None, rtloader_root=None, python_home_2=None, @@ -201,6 +202,10 @@ def get_build_flags( if sys.platform.startswith('linux') and install_path: ldflags += f"-X {REPO_PATH}/pkg/config/setup.InstallPath={install_path} " + # setting the run path + if sys.platform.startswith('linux') and run_path: + ldflags += f"-X {REPO_PATH}/pkg/config/setup.defaultRunPath={run_path} " + # setting python homes in the code if python_home_2: ldflags += f"-X {REPO_PATH}/pkg/collector/python.pythonHome2={python_home_2} " diff --git a/test/new-e2e/tests/installer/linux_test.go b/test/new-e2e/tests/installer/linux_test.go index 5a29e35668b16..0f4e425126b21 100644 --- a/test/new-e2e/tests/installer/linux_test.go +++ b/test/new-e2e/tests/installer/linux_test.go @@ -35,7 +35,7 @@ const ( confDir = "/etc/datadog-agent" logDir = "/var/log/datadog" stableInstallerRunDir = "/opt/datadog-packages/datadog-installer/stable/run" - locksDir = "/var/run/datadog-packages" + locksDir = "/var/run/datadog/installer/locks" packagesDir = "/opt/datadog-packages" bootInstallerDir = "/opt/datadog-installer" rpm = "rpm" @@ -231,7 +231,7 @@ func (v *installerSuite) TestUninstall() { installAssertions := []string{ "test -d /opt/datadog-packages", "test -d /opt/datadog-installer", - "test -d /var/run/datadog-packages", + "test -d /var/run/datadog/installer/locks", "test -L /usr/bin/datadog-installer", "test -L /usr/bin/datadog-bootstrap", } From dda622869d35b8c29be070dd01d82f17d69a043b Mon Sep 17 00:00:00 2001 From: Arthur Bellal Date: Fri, 10 May 2024 16:27:55 +0200 Subject: [PATCH 02/10] remove chown dd-agent on the old run --- .../installer/service/datadog_installer.go | 4 ---- test/new-e2e/tests/installer/linux_test.go | 22 +++++++------------ 2 files changed, 8 insertions(+), 18 deletions(-) diff --git a/pkg/fleet/installer/service/datadog_installer.go b/pkg/fleet/installer/service/datadog_installer.go index ceb2490a83463..7097b4f1c42ad 100644 --- a/pkg/fleet/installer/service/datadog_installer.go +++ b/pkg/fleet/installer/service/datadog_installer.go @@ -102,10 +102,6 @@ func SetupInstaller(ctx context.Context) (err error) { if err != nil { return fmt.Errorf("error changing owner of /var/run/datadog: %w", err) } - err = os.Chown("/opt/datadog-packages/datadog-installer/stable/run", ddAgentUID, ddAgentGID) - if err != nil { - return fmt.Errorf("error changing owner of /opt/datadog-packages/datadog-installer/stable/run: %w", err) - } // Create installer path symlink err = os.Symlink("/opt/datadog-packages/datadog-installer/stable/bin/installer/installer", "/usr/bin/datadog-installer") diff --git a/test/new-e2e/tests/installer/linux_test.go b/test/new-e2e/tests/installer/linux_test.go index 0f4e425126b21..b5dd6e8bda1a9 100644 --- a/test/new-e2e/tests/installer/linux_test.go +++ b/test/new-e2e/tests/installer/linux_test.go @@ -32,15 +32,14 @@ import ( ) const ( - confDir = "/etc/datadog-agent" - logDir = "/var/log/datadog" - stableInstallerRunDir = "/opt/datadog-packages/datadog-installer/stable/run" - locksDir = "/var/run/datadog/installer/locks" - packagesDir = "/opt/datadog-packages" - bootInstallerDir = "/opt/datadog-installer" - rpm = "rpm" - dpkg = "dpkg" - zypper = "zypper" + confDir = "/etc/datadog-agent" + logDir = "/var/log/datadog" + locksDir = "/var/run/datadog/installer/locks" + packagesDir = "/opt/datadog-packages" + bootInstallerDir = "/opt/datadog-installer" + rpm = "rpm" + dpkg = "dpkg" + zypper = "zypper" ) type installerSuite struct { @@ -158,11 +157,6 @@ func (v *installerSuite) TestInstallerDirs() { require.Equal(v.T(), "root\n", host.MustExecute(`stat -c "%U" `+dir)) require.Equal(v.T(), "root\n", host.MustExecute(`stat -c "%G" `+dir)) } - for _, dir := range []string{stableInstallerRunDir} { - require.Equal(v.T(), "dd-agent\n", host.MustExecute(`stat -c "%U" `+dir)) - require.Equal(v.T(), "dd-agent\n", host.MustExecute(`stat -c "%G" `+dir)) - - } require.Equal(v.T(), "drwxrwxrwx\n", host.MustExecute(`stat -c "%A" `+locksDir)) require.Equal(v.T(), "drwxr-xr-x\n", host.MustExecute(`stat -c "%A" `+packagesDir)) } From a10b22489a01bc7bd10312e6ae6462361f433dfa Mon Sep 17 00:00:00 2001 From: Arthur Bellal Date: Sat, 11 May 2024 12:56:17 +0200 Subject: [PATCH 03/10] run py lint --- tasks/installer.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tasks/installer.py b/tasks/installer.py index 286973f4a1af7..cc935a4edfc8c 100644 --- a/tasks/installer.py +++ b/tasks/installer.py @@ -29,7 +29,9 @@ def build( Build the updater. """ - ldflags, gcflags, env = get_build_flags(ctx, major_version=MAJOR_VERSION, install_path=install_path, run_path=run_path) + ldflags, gcflags, env = get_build_flags( + ctx, major_version=MAJOR_VERSION, install_path=install_path, run_path=run_path + ) build_include = ( get_default_build_tags( From f055ba0165ff85b1364ac0e0b376a9c63613262d Mon Sep 17 00:00:00 2001 From: Arthur Bellal Date: Sat, 11 May 2024 15:04:24 +0200 Subject: [PATCH 04/10] fix uninstall test issue --- test/new-e2e/tests/installer/linux_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/new-e2e/tests/installer/linux_test.go b/test/new-e2e/tests/installer/linux_test.go index 9f5124557a0ac..5d5c8233c45aa 100644 --- a/test/new-e2e/tests/installer/linux_test.go +++ b/test/new-e2e/tests/installer/linux_test.go @@ -255,7 +255,7 @@ func (v *installerSuite) TestUninstall() { case zypper: host.MustExecute("sudo zypper --non-interactive install datadog-installer") } - host.MustExecute("sudo /usr/bin/datadog-bootstrap bootstrap") + v.bootstrap(false) for _, assertion := range installAssertions { _ = host.MustExecute(assertion) } From 81b305811e37f3cd686fadab523b4fbc0272a1f8 Mon Sep 17 00:00:00 2001 From: Baptiste Foy Date: Wed, 15 May 2024 10:06:47 +0200 Subject: [PATCH 05/10] trigger tests From db14be0ce0f38c13df44ec6217c77a55c11c2565 Mon Sep 17 00:00:00 2001 From: Baptiste Foy Date: Wed, 15 May 2024 11:22:53 +0200 Subject: [PATCH 06/10] reset linux_test before merge --- test/new-e2e/tests/installer/linux_test.go | 26 +++++++++++++--------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/test/new-e2e/tests/installer/linux_test.go b/test/new-e2e/tests/installer/linux_test.go index 5d5c8233c45aa..1063bb0461de2 100644 --- a/test/new-e2e/tests/installer/linux_test.go +++ b/test/new-e2e/tests/installer/linux_test.go @@ -32,14 +32,15 @@ import ( ) const ( - confDir = "/etc/datadog-agent" - logDir = "/var/log/datadog" - locksDir = "/var/run/datadog/installer/locks" - packagesDir = "/opt/datadog-packages" - bootInstallerDir = "/opt/datadog-installer" - rpm = "rpm" - dpkg = "dpkg" - zypper = "zypper" + confDir = "/etc/datadog-agent" + logDir = "/var/log/datadog" + stableInstallerRunDir = "/opt/datadog-packages/datadog-installer/stable/run" + locksDir = "/var/run/datadog-packages" + packagesDir = "/opt/datadog-packages" + bootInstallerDir = "/opt/datadog-installer" + rpm = "rpm" + dpkg = "dpkg" + zypper = "zypper" ) type installerSuite struct { @@ -158,6 +159,11 @@ func (v *installerSuite) TestInstallerDirs() { require.Equal(v.T(), "root\n", host.MustExecute(`stat -c "%U" `+dir)) require.Equal(v.T(), "root\n", host.MustExecute(`stat -c "%G" `+dir)) } + for _, dir := range []string{stableInstallerRunDir} { + require.Equal(v.T(), "dd-agent\n", host.MustExecute(`stat -c "%U" `+dir)) + require.Equal(v.T(), "dd-agent\n", host.MustExecute(`stat -c "%G" `+dir)) + + } require.Equal(v.T(), "drwxrwxrwx\n", host.MustExecute(`stat -c "%A" `+locksDir)) require.Equal(v.T(), "drwxr-xr-x\n", host.MustExecute(`stat -c "%A" `+packagesDir)) } @@ -227,7 +233,7 @@ func (v *installerSuite) TestUninstall() { installAssertions := []string{ "test -d /opt/datadog-packages", "test -d /opt/datadog-installer", - "test -d /var/run/datadog/installer/locks", + "test -d /var/run/datadog-packages", "test -L /usr/bin/datadog-installer", "test -L /usr/bin/datadog-bootstrap", } @@ -255,7 +261,7 @@ func (v *installerSuite) TestUninstall() { case zypper: host.MustExecute("sudo zypper --non-interactive install datadog-installer") } - v.bootstrap(false) + host.MustExecute("sudo /usr/bin/datadog-bootstrap bootstrap") for _, assertion := range installAssertions { _ = host.MustExecute(assertion) } From 041fe961b4d1b7bc89822fb6ad8e33ca30283077 Mon Sep 17 00:00:00 2001 From: Baptiste Foy Date: Wed, 15 May 2024 11:35:13 +0200 Subject: [PATCH 07/10] adapt E2E --- test/new-e2e/tests/installer/package_installer_test.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/test/new-e2e/tests/installer/package_installer_test.go b/test/new-e2e/tests/installer/package_installer_test.go index 96632393deac0..05925b67b9e5f 100644 --- a/test/new-e2e/tests/installer/package_installer_test.go +++ b/test/new-e2e/tests/installer/package_installer_test.go @@ -35,12 +35,14 @@ func (s *packageInstallerSuite) TestInstall() { state.AssertUserHasGroup("dd-agent", "dd-agent") state.AssertDirExists("/var/log/datadog", 0755, "dd-agent", "dd-agent") - state.AssertDirExists("/var/run/datadog-packages", 0777, "root", "root") + state.AssertDirExists("/var/run/datadog", 0755, "dd-agent", "dd-agent") + state.AssertDirExists("/var/run/datadog/installer", 0755, "dd-agent", "dd-agent") + state.AssertDirExists("/var/run/datadog/installer/locks", 0777, "root", "root") state.AssertDirExists("/opt/datadog-installer", 0755, "root", "root") state.AssertDirExists("/opt/datadog-packages", 0755, "root", "root") state.AssertDirExists("/opt/datadog-packages/datadog-installer", 0755, "root", "root") - state.AssertDirExists("/opt/datadog-packages/datadog-installer/stable/run", 0755, "dd-agent", "dd-agent") + state.AssertDirExists("/opt/datadog-packages/datadog-installer/stable/run", 0755, "root", "root") state.AssertSymlinkExists("/usr/bin/datadog-bootstrap", "/opt/datadog-installer/bin/installer/installer", "root", "root") state.AssertSymlinkExists("/usr/bin/datadog-installer", "/opt/datadog-packages/datadog-installer/stable/bin/installer/installer", "root", "root") @@ -63,7 +65,7 @@ func (s *packageInstallerSuite) TestUninstall() { // state that should get removed state.AssertPathDoesNotExist("/opt/datadog-installer") - state.AssertPathDoesNotExist("/var/run/datadog-packages") + state.AssertPathDoesNotExist("/var/run/datadog/installer") state.AssertPathDoesNotExist("/opt/datadog-packages") state.AssertPathDoesNotExist("/usr/bin/datadog-bootstrap") From 078cf095a971605745434e6f18246692ac435092 Mon Sep 17 00:00:00 2001 From: Arthur Bellal Date: Wed, 15 May 2024 13:17:06 +0200 Subject: [PATCH 08/10] minor e2e test changes --- test/new-e2e/tests/installer/package_installer_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/new-e2e/tests/installer/package_installer_test.go b/test/new-e2e/tests/installer/package_installer_test.go index 05925b67b9e5f..f747f31964717 100644 --- a/test/new-e2e/tests/installer/package_installer_test.go +++ b/test/new-e2e/tests/installer/package_installer_test.go @@ -42,7 +42,6 @@ func (s *packageInstallerSuite) TestInstall() { state.AssertDirExists("/opt/datadog-installer", 0755, "root", "root") state.AssertDirExists("/opt/datadog-packages", 0755, "root", "root") state.AssertDirExists("/opt/datadog-packages/datadog-installer", 0755, "root", "root") - state.AssertDirExists("/opt/datadog-packages/datadog-installer/stable/run", 0755, "root", "root") state.AssertSymlinkExists("/usr/bin/datadog-bootstrap", "/opt/datadog-installer/bin/installer/installer", "root", "root") state.AssertSymlinkExists("/usr/bin/datadog-installer", "/opt/datadog-packages/datadog-installer/stable/bin/installer/installer", "root", "root") @@ -62,6 +61,7 @@ func (s *packageInstallerSuite) TestUninstall() { state.AssertUserHasGroup("dd-agent", "dd-agent") state.AssertDirExists("/var/log/datadog", 0755, "dd-agent", "dd-agent") + state.AssertDirExists("/var/run/datadog", 0755, "dd-agent", "dd-agent") // state that should get removed state.AssertPathDoesNotExist("/opt/datadog-installer") From cf04f8f68f7c540ac645aa7c5b064cb3080924ca Mon Sep 17 00:00:00 2001 From: Arthur Bellal Date: Wed, 15 May 2024 14:49:01 +0200 Subject: [PATCH 09/10] Update pkg/fleet/installer/service/datadog_installer.go Co-authored-by: Florent Clarret --- pkg/fleet/installer/service/datadog_installer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/fleet/installer/service/datadog_installer.go b/pkg/fleet/installer/service/datadog_installer.go index ca35e075f83d4..801d1d9d590c0 100644 --- a/pkg/fleet/installer/service/datadog_installer.go +++ b/pkg/fleet/installer/service/datadog_installer.go @@ -90,7 +90,7 @@ func SetupInstaller(ctx context.Context) (err error) { } err = os.Chown("/var/run/datadog/installer", ddAgentUID, ddAgentGID) if err != nil { - return fmt.Errorf("error changing owner of /var/run/datadog: %w", err) + return fmt.Errorf("error changing owner of /var/run/datadog/installer: %w", err) } // Create installer path symlink From e4b3ffbe8fdf50eceaf5fcd250d33de86249b302 Mon Sep 17 00:00:00 2001 From: Arthur Bellal Date: Wed, 15 May 2024 14:49:09 +0200 Subject: [PATCH 10/10] Update pkg/fleet/installer/service/datadog_installer.go Co-authored-by: Kevin Fairise <132568982+KevinFairise2@users.noreply.github.com> --- pkg/fleet/installer/service/datadog_installer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/fleet/installer/service/datadog_installer.go b/pkg/fleet/installer/service/datadog_installer.go index 801d1d9d590c0..e5be33d2ab27b 100644 --- a/pkg/fleet/installer/service/datadog_installer.go +++ b/pkg/fleet/installer/service/datadog_installer.go @@ -69,7 +69,7 @@ func SetupInstaller(ctx context.Context) (err error) { } err = os.MkdirAll("/var/run/datadog/installer", 0755) if err != nil { - return fmt.Errorf("error creating /var/log/datadog: %w", err) + return fmt.Errorf("error creating /var/run/datadog/installer: %w", err) } err = os.MkdirAll("/var/run/datadog/installer/locks", 0777) if err != nil {