From 0b287911d83704306a87cbe759c2de5d63dc0691 Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Tue, 11 Nov 2025 11:18:38 +0100 Subject: [PATCH 1/4] Fix update of ldcache on non-debian containers This change ensures that the ldcache in a non-debian container includes libraries at /lib64 and /usr/lib64 when running on debian host. This is required because the system search paths do not include these folders by default resulting in a non-debian container missing system libraries from the ldcache. Signed-off-by: Evan Lezar --- internal/ldconfig/ldconfig.go | 37 ++++++++++++++++------ internal/ldconfig/ldconfig_test.go | 2 +- tests/e2e/nvidia-container-toolkit_test.go | 19 +++++++++++ 3 files changed, 48 insertions(+), 10 deletions(-) diff --git a/internal/ldconfig/ldconfig.go b/internal/ldconfig/ldconfig.go index 491c69f8c..2291be8e3 100644 --- a/internal/ldconfig/ldconfig.go +++ b/internal/ldconfig/ldconfig.go @@ -67,7 +67,8 @@ func NewRunner(id string, ldconfigPath string, containerRoot string, additionala // This struct is used to perform operations on the ldcache and libraries in a // particular root (e.g. a container). // -// args[0] is the reexec initializer function name +// args[0] is the reexec initializer function name and is required. +// // The following flags are required: // // --ldconfig-path=LDCONFIG_PATH the path to ldconfig on the host @@ -76,16 +77,20 @@ func NewRunner(id string, ldconfigPath string, containerRoot string, additionala // The following flags are optional: // // --is-debian-like-host Indicates that the host system is debian-based. +// See https://github.com/NVIDIA/nvidia-container-toolkit/pull/1444 // // The remaining args are folders where soname symlinks need to be created. func NewFromArgs(args ...string) (*Ldconfig, error) { if len(args) < 1 { return nil, fmt.Errorf("incorrect arguments: %v", args) } - fs := flag.NewFlagSet(args[1], flag.ExitOnError) + fs := flag.NewFlagSet("ldconfig-options", flag.ExitOnError) ldconfigPath := fs.String("ldconfig-path", "", "the path to ldconfig on the host") containerRoot := fs.String("container-root", "", "the path in which ldconfig must be run") - isDebianLikeHost := fs.Bool("is-debian-like-host", false, "the hook is running from a Debian-like host") + isDebianLikeHost := fs.Bool("is-debian-like-host", false, `indicates that the host system is debian-based. +This allows us to handle the case where there are differences in behavior +between the ldconfig from the host (as executed from an update-ldcache hook) and +ldconfig in the container. Such differences include system search paths.`) if err := fs.Parse(args[1:]); err != nil { return nil, err } @@ -116,7 +121,7 @@ func (l *Ldconfig) UpdateLDCache() error { // Explicitly specify using /etc/ld.so.conf since the host's ldconfig may // be configured to use a different config file by default. const topLevelLdsoconfFilePath = "/etc/ld.so.conf" - filteredDirectories, err := l.filterDirectories(topLevelLdsoconfFilePath, l.directories...) + filteredDirectories, ldconfigDirs, err := l.filterDirectories(topLevelLdsoconfFilePath, l.directories...) if err != nil { return err } @@ -126,6 +131,19 @@ func (l *Ldconfig) UpdateLDCache() error { "-f", topLevelLdsoconfFilePath, "-C", "/etc/ld.so.cache", } + // If we are running in a non-debian container on a debian host we also + // need to add the system directories for non-debian hosts to the list of + // folders processed by ldconfig. + // We only do this if they are not already tracked, since the folders on + // on the command line have a higher priority than folders in ld.so.conf. + if l.isDebianLikeHost && !l.isDebianLikeContainer { + for _, systemSearchPath := range l.getSystemSearchPaths() { + if _, ok := ldconfigDirs[systemSearchPath]; ok { + continue + } + args = append(args, "/lib64", "/usr/lib64") + } + } if err := createLdsoconfdFile(ldsoconfdFilenamePattern, filteredDirectories...); err != nil { return fmt.Errorf("failed to update ld.so.conf.d: %w", err) @@ -157,10 +175,10 @@ func (l *Ldconfig) prepareRoot() (string, error) { return ldconfigPath, nil } -func (l *Ldconfig) filterDirectories(configFilePath string, directories ...string) ([]string, error) { +func (l *Ldconfig) filterDirectories(configFilePath string, directories ...string) ([]string, map[string]struct{}, error) { ldconfigDirs, err := l.getLdsoconfDirectories(configFilePath) if err != nil { - return nil, err + return nil, nil, err } var filtered []string @@ -169,8 +187,9 @@ func (l *Ldconfig) filterDirectories(configFilePath string, directories ...strin continue } filtered = append(filtered, d) + ldconfigDirs[d] = struct{}{} } - return filtered, nil + return filtered, ldconfigDirs, nil } // createLdsoconfdFile creates a file at /etc/ld.so.conf.d/. @@ -218,7 +237,7 @@ func createLdsoconfdFile(pattern string, dirs ...string) error { // files that refer to the directory. func (l *Ldconfig) getLdsoconfDirectories(configFilePath string) (map[string]struct{}, error) { ldconfigDirs := make(map[string]struct{}) - for _, d := range l.getSystemSerachPaths() { + for _, d := range l.getSystemSearchPaths() { ldconfigDirs[d] = struct{}{} } @@ -247,7 +266,7 @@ func (l *Ldconfig) getLdsoconfDirectories(configFilePath string) (map[string]str return ldconfigDirs, nil } -func (l *Ldconfig) getSystemSerachPaths() []string { +func (l *Ldconfig) getSystemSearchPaths() []string { if l.isDebianLikeContainer { debianSystemSearchPaths() } diff --git a/internal/ldconfig/ldconfig_test.go b/internal/ldconfig/ldconfig_test.go index a22876719..308faa0ec 100644 --- a/internal/ldconfig/ldconfig_test.go +++ b/internal/ldconfig/ldconfig_test.go @@ -117,7 +117,7 @@ include INCLUDED_PATTERN* l := &Ldconfig{ isDebianLikeContainer: true, } - filtered, err := l.filterDirectories(topLevelConfPath, tc.input...) + filtered, _, err := l.filterDirectories(topLevelConfPath, tc.input...) require.NoError(t, err) require.Equal(t, tc.expected, filtered) diff --git a/tests/e2e/nvidia-container-toolkit_test.go b/tests/e2e/nvidia-container-toolkit_test.go index 69086301b..33181345e 100644 --- a/tests/e2e/nvidia-container-toolkit_test.go +++ b/tests/e2e/nvidia-container-toolkit_test.go @@ -570,4 +570,23 @@ EOF`) Expect(output).To(Equal(expectedOutput)) }) }) + + When("running a ubi9 container", Ordered, func() { + var ( + expectedOutput string + ) + BeforeAll(func(ctx context.Context) { + _, _, err := runner.Run(`docker pull redhat/ubi9`) + Expect(err).ToNot(HaveOccurred()) + + expectedOutput, _, err = runner.Run(`docker run --rm --runtime=runc redhat/ubi9 bash -c "ldconfig -p | grep libc.so."`) + Expect(err).ToNot(HaveOccurred()) + }) + + It("should include the system libraries when using the nvidia-container-runtime", func(ctx context.Context) { + output, _, err := runner.Run(`docker run --rm --runtime=nvidia -e NVIDIA_VISIBLE_DEVICES=all redhat/ubi9 bash -c "ldconfig -p | grep libc.so."`) + Expect(err).ToNot(HaveOccurred()) + Expect(output).To(Equal(expectedOutput)) + }) + }) }) From 1cf8ccd5a7087a2502710fde42c43c9bd75c7348 Mon Sep 17 00:00:00 2001 From: Jean-Francois Roy Date: Fri, 14 Nov 2025 11:17:24 -0800 Subject: [PATCH 2/4] Add missing return in `getSystemSearchPaths` for debian like containers Signed-off-by: Evan Lezar --- internal/ldconfig/ldconfig.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/ldconfig/ldconfig.go b/internal/ldconfig/ldconfig.go index 2291be8e3..4d55a8391 100644 --- a/internal/ldconfig/ldconfig.go +++ b/internal/ldconfig/ldconfig.go @@ -268,7 +268,7 @@ func (l *Ldconfig) getLdsoconfDirectories(configFilePath string) (map[string]str func (l *Ldconfig) getSystemSearchPaths() []string { if l.isDebianLikeContainer { - debianSystemSearchPaths() + return debianSystemSearchPaths() } return nonDebianSystemSearchPaths() } From f432507185908d3d00fd74502fdb85dee5e70675 Mon Sep 17 00:00:00 2001 From: Jean-Francois Roy Date: Thu, 13 Nov 2025 16:56:21 -0800 Subject: [PATCH 3/4] ldconfig: Determine container "debian-ness" after root pivot Signed-off-by: Evan Lezar --- internal/ldconfig/ldconfig.go | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/internal/ldconfig/ldconfig.go b/internal/ldconfig/ldconfig.go index 4d55a8391..becc3bd50 100644 --- a/internal/ldconfig/ldconfig.go +++ b/internal/ldconfig/ldconfig.go @@ -103,11 +103,10 @@ ldconfig in the container. Such differences include system search paths.`) } l := &Ldconfig{ - ldconfigPath: *ldconfigPath, - inRoot: *containerRoot, - isDebianLikeHost: *isDebianLikeHost, - isDebianLikeContainer: isDebian(), - directories: fs.Args(), + ldconfigPath: *ldconfigPath, + inRoot: *containerRoot, + isDebianLikeHost: *isDebianLikeHost, + directories: fs.Args(), } return l, nil } @@ -118,6 +117,9 @@ func (l *Ldconfig) UpdateLDCache() error { return err } + // `prepareRoot` pivots to the container root, so can now set the container "debian-ness". + l.isDebianLikeContainer = isDebian() + // Explicitly specify using /etc/ld.so.conf since the host's ldconfig may // be configured to use a different config file by default. const topLevelLdsoconfFilePath = "/etc/ld.so.conf" From d7f94e683becbcde9825707f51d37edf03167726 Mon Sep 17 00:00:00 2001 From: Jean-Francois Roy Date: Mon, 17 Nov 2025 13:09:47 -0800 Subject: [PATCH 4/4] Append system paths at the end of top-level ld.so.conf Signed-off-by: Evan Lezar --- internal/ldconfig/ldconfig.go | 107 +++++++++++++++++++++-------- internal/ldconfig/ldconfig_test.go | 72 ++++++++++++++++++- 2 files changed, 151 insertions(+), 28 deletions(-) diff --git a/internal/ldconfig/ldconfig.go b/internal/ldconfig/ldconfig.go index becc3bd50..33021c056 100644 --- a/internal/ldconfig/ldconfig.go +++ b/internal/ldconfig/ldconfig.go @@ -123,7 +123,7 @@ func (l *Ldconfig) UpdateLDCache() error { // Explicitly specify using /etc/ld.so.conf since the host's ldconfig may // be configured to use a different config file by default. const topLevelLdsoconfFilePath = "/etc/ld.so.conf" - filteredDirectories, ldconfigDirs, err := l.filterDirectories(topLevelLdsoconfFilePath, l.directories...) + filteredDirectories, err := l.filterDirectories(topLevelLdsoconfFilePath, l.directories...) if err != nil { return err } @@ -133,24 +133,21 @@ func (l *Ldconfig) UpdateLDCache() error { "-f", topLevelLdsoconfFilePath, "-C", "/etc/ld.so.cache", } - // If we are running in a non-debian container on a debian host we also - // need to add the system directories for non-debian hosts to the list of - // folders processed by ldconfig. - // We only do this if they are not already tracked, since the folders on - // on the command line have a higher priority than folders in ld.so.conf. - if l.isDebianLikeHost && !l.isDebianLikeContainer { - for _, systemSearchPath := range l.getSystemSearchPaths() { - if _, ok := ldconfigDirs[systemSearchPath]; ok { - continue - } - args = append(args, "/lib64", "/usr/lib64") - } - } if err := createLdsoconfdFile(ldsoconfdFilenamePattern, filteredDirectories...); err != nil { return fmt.Errorf("failed to update ld.so.conf.d: %w", err) } + // In most cases, the hook will be executing a host ldconfig that may be configured widely + // differently from what the container image expects. The common case is Debian vs non-Debian. + // But there are also hosts that configure ldconfig to search in a glibc prefix + // (e.g. /usr/lib/glibc). To avoid all these cases, append the container's expected system + // search paths to the top-level ld.so.conf. This will ensure they get scanned but won't + // materially change the scan order. + if err := appendSystemSearchPathsToLdsoconf(topLevelLdsoconfFilePath, l.getSystemSearchPaths()...); err != nil { + return fmt.Errorf("failed to append system search paths to %s: %w", topLevelLdsoconfFilePath, err) + } + return SafeExec(ldconfigPath, args, nil) } @@ -177,10 +174,10 @@ func (l *Ldconfig) prepareRoot() (string, error) { return ldconfigPath, nil } -func (l *Ldconfig) filterDirectories(configFilePath string, directories ...string) ([]string, map[string]struct{}, error) { +func (l *Ldconfig) filterDirectories(configFilePath string, directories ...string) ([]string, error) { ldconfigDirs, err := l.getLdsoconfDirectories(configFilePath) if err != nil { - return nil, nil, err + return nil, err } var filtered []string @@ -191,7 +188,7 @@ func (l *Ldconfig) filterDirectories(configFilePath string, directories ...strin filtered = append(filtered, d) ldconfigDirs[d] = struct{}{} } - return filtered, ldconfigDirs, nil + return filtered, nil } // createLdsoconfdFile creates a file at /etc/ld.so.conf.d/. @@ -235,6 +232,24 @@ func createLdsoconfdFile(pattern string, dirs ...string) error { return nil } +func appendSystemSearchPathsToLdsoconf(configFilePath string, dirs ...string) error { + if len(dirs) == 0 { + return nil + } + configFile, err := os.OpenFile(configFilePath, os.O_APPEND|os.O_WRONLY, 0644) + if err != nil { + return fmt.Errorf("failed to open config file: %w", err) + } + defer configFile.Close() + for _, dir := range dirs { + _, err = fmt.Fprintf(configFile, "%s\n", dir) + if err != nil { + return fmt.Errorf("failed to update config file: %w", err) + } + } + return nil +} + // getLdsoconfDirectories returns a map of ldsoconf directories to the conf // files that refer to the directory. func (l *Ldconfig) getLdsoconfDirectories(configFilePath string) (map[string]struct{}, error) { @@ -318,22 +333,61 @@ func isDebian() bool { return !info.IsDir() } -// nonDebianSystemSearchPaths returns the system search paths for non-Debian -// systems. +// nonDebianSystemSearchPaths returns the system search paths for non-Debian systems. +// +// glibc ldconfig's calls `add_system_dir` with `SLIBDIR` and `LIBDIR` (if they are not equal). On +// aarch64 and x86_64, `add_system_dir` is a macro that scans the provided path. If the path ends +// with "/lib64" (or "/libx32", x86_64 only), it strips those suffixes. Then it registers the +// resulting path. Then if the path ends with "/lib", it registers "path"+"64" (and "path"+"x32", +// x86_64 only). +// +// By default, "LIBDIR" is "/usr/lib" and "SLIBDIR" is "/lib". Note that on modern distributions, +// "/lib" is usually a symlink to "/usr/lib" and "/lib64" to "/usr/lib64". ldconfig resolves +// symlinks and skips duplicate directory entries. +// +// To get the list of system paths, you can invoke the dynamic linker with `--list-diagnostics` and +// look for "path.system_dirs". For example +// `docker run --rm -ti fedora:latest /lib64/ld-linux-x86-64.so.2 --list-diagnostics | grep path.system_dirs`. // -// This list was taken from the output of: +// On most distributions, including Fedora and derivatives, this yields the following +// ldconfig system search paths. // -// docker run --rm -ti redhat/ubi9 /usr/lib/ld-linux-aarch64.so.1 --help | grep -A6 "Shared library search path" +// TODO: Add other architectures that have custom `add_system_dir` macros (e.g. riscv) +// TODO: Replace with executing the container's dynamlic linker with `--list-diagnostics`? func nonDebianSystemSearchPaths() []string { - return []string{"/lib64", "/usr/lib64"} + var paths []string + paths = append(paths, "/lib", "/usr/lib") + switch runtime.GOARCH { + case "amd64": + paths = append(paths, + "/lib/lib64", + "/usr/lib64", + "/libx32", + "/usr/libx32", + ) + case "arm64": + paths = append(paths, + "/lib/lib64", + "/usr/lib64", + ) + } + return paths } -// debianSystemSearchPaths returns the system search paths for Debian-like -// systems. +// debianSystemSearchPaths returns the system search paths for Debian-like systems. +// +// Debian (and derivatives) apply their multi-arch patch to glibc, which modifies ldconfig to +// use the same set of system paths as the dynamic linker. These paths are going to include the +// multi-arch directory _and_ by default "/lib" and "/usr/lib" for compatibility. // -// This list was taken from the output of: +// To get the list of system paths, you can invoke the dynamic linker with `--list-diagnostics` and +// look for "path.system_dirs". For example +// `docker run --rm -ti ubuntu:latest /lib64/ld-linux-x86-64.so.2 --list-diagnostics | grep path.system_dirs`. // -// docker run --rm -ti ubuntu /usr/lib/aarch64-linux-gnu/ld-linux-aarch64.so.1 --help | grep -A6 "Shared library search path" +// This yields the following ldconfig system search paths. +// +// TODO: Add other architectures that have custom `add_system_dir` macros (e.g. riscv) +// TODO: Replace with executing the container's dynamlic linker with `--list-diagnostics`? func debianSystemSearchPaths() []string { var paths []string switch runtime.GOARCH { @@ -349,6 +403,5 @@ func debianSystemSearchPaths() []string { ) } paths = append(paths, "/lib", "/usr/lib") - return paths } diff --git a/internal/ldconfig/ldconfig_test.go b/internal/ldconfig/ldconfig_test.go index 308faa0ec..4b7bb3f4a 100644 --- a/internal/ldconfig/ldconfig_test.go +++ b/internal/ldconfig/ldconfig_test.go @@ -19,6 +19,7 @@ package ldconfig import ( "os" + "path/filepath" "strings" "testing" @@ -117,10 +118,79 @@ include INCLUDED_PATTERN* l := &Ldconfig{ isDebianLikeContainer: true, } - filtered, _, err := l.filterDirectories(topLevelConfPath, tc.input...) + filtered, err := l.filterDirectories(topLevelConfPath, tc.input...) require.NoError(t, err) require.Equal(t, tc.expected, filtered) }) } } + +func TestAppendSystemSearchPathsToLdsoconf(t *testing.T) { + testCases := []struct { + description string + initialContent string + noFile bool + dirs []string + expectedFinal string + expectError bool + }{ + { + description: "append to empty file", + initialContent: "", + dirs: []string{"/lib", "/usr/lib"}, + expectedFinal: "/lib\n/usr/lib\n", + }, + { + description: "append to existing content", + initialContent: "# existing config\n/existing/path\n", + dirs: []string{"/lib", "/usr/lib"}, + expectedFinal: "# existing config\n/existing/path\n/lib\n/usr/lib\n", + }, + { + description: "append empty does nothing", + initialContent: "# existing config\n", + dirs: []string{}, + expectedFinal: "# existing config\n", + }, + { + description: "append empty does not create file", + noFile: true, + dirs: []string{}, + }, + { + description: "append to non-existent file fails", + noFile: true, + dirs: []string{"/lib"}, + expectError: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.description, func(t *testing.T) { + tmpDir := t.TempDir() + configPath := filepath.Join(tmpDir, "ld.so.conf") + + if !tc.noFile { + err := os.WriteFile(configPath, []byte(tc.initialContent), 0600) + require.NoError(t, err) + } + + err := appendSystemSearchPathsToLdsoconf(configPath, tc.dirs...) + if tc.expectError { + require.Error(t, err) + return + } + require.NoError(t, err) + + if !tc.noFile { + content, err := os.ReadFile(configPath) + require.NoError(t, err) + require.Equal(t, tc.expectedFinal, string(content)) + } else { + _, err := os.Stat(configPath) + require.True(t, os.IsNotExist(err)) + } + }) + } +}