fix: ensure source baseenv not overwritten in local build (release-4.2)

.circleci/config.yml

@@ -6,7 +6,7 @@ orbs:
 parameters:
   go-version:
     type: string
-    default: '1.22.7'
+    default: '1.23.0'
 
 executors:
   node:

CHANGELOG.md

@@ -1,5 +1,12 @@
 # SingularityCE Changelog
 
+## Changes Since Last Release
+
+### Bug Fixes
+
+- Fix regression from 4.1.5 that overwrites source image runscript, environment
+  etc. in build from local image.
+
 ## 4.2.1 \[2024-09-13\]
 
 ### Bug Fixes

e2e/build/regressions.go

@@ -637,14 +637,35 @@ func (c *imgBuildTests) issue2607(t *testing.T) {
 }
 
 // Check that the build process from an image doesn't fail when the source image
-// includes symlinks.
+// includes symlinks. Also confirm that the base environment files from the
+// source image are not overwritten (#3353).
 func (c *imgBuildTests) issue3084(t *testing.T) {
+	e2e.EnsureImage(t, c.env)
 	require.Command(t, "mksquashfs")
 
+	// Extract standard test image to a sandbox dir.
 	rootfs := filepath.Join(c.env.TestDir, "issue_3084_rootfs")
 	if err := os.Mkdir(rootfs, 0o755); err != nil {
 		t.Fatal(err)
 	}
+	c.env.RunSingularity(
+		t,
+		e2e.WithProfile(e2e.UserProfile),
+		e2e.WithCommand("build"),
+		e2e.WithArgs("--force", "--sandbox", rootfs, c.env.ImagePath),
+		e2e.ExpectExit(
+			0,
+		),
+	)
+
+	// Remove existing `/tmp`, `/var`. Create `/var/tmp` -> `/tmp` and `/var/log` ->
+	// `/tmp` symlinks that cause unsquashfs extraction issue.
+	if err := os.RemoveAll(filepath.Join(rootfs, "tmp")); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.RemoveAll(filepath.Join(rootfs, "var")); err != nil {
+		t.Fatal(err)
+	}
 	if err := os.Mkdir(filepath.Join(rootfs, "tmp"), 0o755); err != nil {
 		t.Fatal(err)
 	}
@@ -657,24 +678,41 @@ func (c *imgBuildTests) issue3084(t *testing.T) {
 	if err := os.Symlink(filepath.Join(rootfs, "tmp"), filepath.Join(rootfs, "var", "log")); err != nil {
 		t.Fatal(err)
 	}
+
+	// Build from resulting structure as a squashfs
 	image := filepath.Join(c.env.TestDir, "issue_3084.img")
 	if err := squashfs.Mksquashfs([]string{rootfs}, image); err != nil {
 		t.Fatal(err)
 	}
-
 	destImage := filepath.Join(c.env.TestDir, "issue_3084_dest.img")
 	c.env.RunSingularity(
 		t,
 		e2e.WithProfile(e2e.RootProfile),
 		e2e.WithCommand("build"),
 		e2e.WithArgs(destImage, image),
+		e2e.ExpectExit(
+			0,
+		),
 		e2e.PostRun(func(_ *testing.T) {
-			os.Remove(destImage)
 			os.Remove(image)
 			os.RemoveAll(rootfs)
 		}),
+	)
+
+	// https://github.com/sylabs/singularity/issues/3353
+	// The source test image runscript outputs "Running command: $*" Make sure
+	// we see it... the runscript should still be in place.
+	c.env.RunSingularity(
+		t,
+		e2e.WithProfile(e2e.UserProfile),
+		e2e.WithCommand("run"),
+		e2e.WithArgs(destImage, "/bin/true"),
 		e2e.ExpectExit(
 			0,
+			e2e.ExpectOutput(e2e.ContainMatch, "Running command: /bin/true"),
 		),
+		e2e.PostRun(func(_ *testing.T) {
+			os.Remove(destImage)
+		}),
 	)
 }

internal/pkg/build/sources/base_environment.go

@@ -1,4 +1,4 @@
-// Copyright (c) 2018-2021, Sylabs Inc. All rights reserved.
+// Copyright (c) 2018-2024, Sylabs Inc. All rights reserved.
 // This software is licensed under a 3-clause BSD license. Please consult the
 // LICENSE.md file distributed with the sources of this project regarding your
 // rights to use or distribute this software.
@@ -343,14 +343,18 @@ func makeSymlinks(rootPath string) error {
 	return nil
 }
 
-func makeFile(name string, perm os.FileMode, s string) (err error) {
+func makeFile(name string, perm os.FileMode, s string, overwrite bool) (err error) {
 	// #4532 - If the file already exists ensure it has requested permissions
 	// as OpenFile won't set on an existing file and some docker
 	// containers have hosts or resolv.conf without write perm.
 	if fs.IsFile(name) {
 		if err = os.Chmod(name, perm); err != nil {
 			return
 		}
+		if !overwrite {
+			sylog.Debugf("not writing to %s - file exists, overwrite is false", name)
+			return
+		}
 	}
 	// Create the file if it's not in the container, or truncate and write s
 	// into it otherwise.
@@ -364,50 +368,55 @@ func makeFile(name string, perm os.FileMode, s string) (err error) {
 	return
 }
 
-func makeFiles(rootPath string) error {
-	if err := makeFile(filepath.Join(rootPath, "etc", "hosts"), 0o644, ""); err != nil {
+func makeFiles(rootPath string, overwrite bool) error {
+	if err := makeFile(filepath.Join(rootPath, "etc", "hosts"), 0o644, "", overwrite); err != nil {
 		return err
 	}
-	if err := makeFile(filepath.Join(rootPath, "etc", "resolv.conf"), 0o644, ""); err != nil {
+	if err := makeFile(filepath.Join(rootPath, "etc", "resolv.conf"), 0o644, "", overwrite); err != nil {
 		return err
 	}
-	if err := makeFile(filepath.Join(rootPath, ".singularity.d", "actions", "exec"), 0o755, execFileContent); err != nil {
+	if err := makeFile(filepath.Join(rootPath, ".singularity.d", "actions", "exec"), 0o755, execFileContent, overwrite); err != nil {
 		return err
 	}
-	if err := makeFile(filepath.Join(rootPath, ".singularity.d", "actions", "run"), 0o755, runFileContent); err != nil {
+	if err := makeFile(filepath.Join(rootPath, ".singularity.d", "actions", "run"), 0o755, runFileContent, overwrite); err != nil {
 		return err
 	}
-	if err := makeFile(filepath.Join(rootPath, ".singularity.d", "actions", "shell"), 0o755, shellFileContent); err != nil {
+	if err := makeFile(filepath.Join(rootPath, ".singularity.d", "actions", "shell"), 0o755, shellFileContent, overwrite); err != nil {
 		return err
 	}
-	if err := makeFile(filepath.Join(rootPath, ".singularity.d", "actions", "start"), 0o755, startFileContent); err != nil {
+	if err := makeFile(filepath.Join(rootPath, ".singularity.d", "actions", "start"), 0o755, startFileContent, overwrite); err != nil {
 		return err
 	}
-	if err := makeFile(filepath.Join(rootPath, ".singularity.d", "actions", "test"), 0o755, testFileContent); err != nil {
+	if err := makeFile(filepath.Join(rootPath, ".singularity.d", "actions", "test"), 0o755, testFileContent, overwrite); err != nil {
 		return err
 	}
-	if err := makeFile(filepath.Join(rootPath, ".singularity.d", "env", "01-base.sh"), 0o755, baseShFileContent); err != nil {
+	if err := makeFile(filepath.Join(rootPath, ".singularity.d", "env", "01-base.sh"), 0o755, baseShFileContent, overwrite); err != nil {
 		return err
 	}
-	if err := makeFile(filepath.Join(rootPath, ".singularity.d", "env", "90-environment.sh"), 0o755, environmentShFileContent); err != nil {
+	if err := makeFile(filepath.Join(rootPath, ".singularity.d", "env", "90-environment.sh"), 0o755, environmentShFileContent, overwrite); err != nil {
 		return err
 	}
-	if err := makeFile(filepath.Join(rootPath, ".singularity.d", "env", "95-apps.sh"), 0o755, appsShFileContent); err != nil {
+	if err := makeFile(filepath.Join(rootPath, ".singularity.d", "env", "95-apps.sh"), 0o755, appsShFileContent, overwrite); err != nil {
 		return err
 	}
-	if err := makeFile(filepath.Join(rootPath, ".singularity.d", "env", "99-base.sh"), 0o755, base99ShFileContent); err != nil {
+	if err := makeFile(filepath.Join(rootPath, ".singularity.d", "env", "99-base.sh"), 0o755, base99ShFileContent, overwrite); err != nil {
 		return err
 	}
-	if err := makeFile(filepath.Join(rootPath, ".singularity.d", "env", "99-runtimevars.sh"), 0o755, base99runtimevarsShFileContent); err != nil {
+	if err := makeFile(filepath.Join(rootPath, ".singularity.d", "env", "99-runtimevars.sh"), 0o755, base99runtimevarsShFileContent, overwrite); err != nil {
 		return err
 	}
-	if err := makeFile(filepath.Join(rootPath, ".singularity.d", "runscript"), 0o755, runscriptFileContent); err != nil {
+	if err := makeFile(filepath.Join(rootPath, ".singularity.d", "runscript"), 0o755, runscriptFileContent, overwrite); err != nil {
 		return err
 	}
-	return makeFile(filepath.Join(rootPath, ".singularity.d", "startscript"), 0o755, startscriptFileContent)
+	return makeFile(filepath.Join(rootPath, ".singularity.d", "startscript"), 0o755, startscriptFileContent, overwrite)
 }
 
-func makeBaseEnv(rootPath string) (err error) {
+// makeBaseEnv inserts Singularity specific directories, symlinks, and files
+// into the contatiner rootfs. If overwrite is true, then any existing files
+// will be overwritten with new content. If overwrite is false, existing files
+// (e.g. where the rootfs has been extracted from an existing image) will not be
+// modified.
+func makeBaseEnv(rootPath string, overwrite bool) (err error) {
 	var info os.FileInfo
 
 	// Ensure we can write into the root of rootPath
@@ -431,7 +440,7 @@ func makeBaseEnv(rootPath string) (err error) {
 		err = fmt.Errorf("build: failed to make environment symlinks: %v", err)
 		return err
 	}
-	if err = makeFiles(rootPath); err != nil {
+	if err = makeFiles(rootPath, overwrite); err != nil {
 		err = fmt.Errorf("build: failed to make environment files: %v", err)
 		return err
 	}

internal/pkg/build/sources/base_environment_test.go

@@ -1,4 +1,4 @@
-// Copyright (c) 2018, Sylabs Inc. All rights reserved.
+// Copyright (c) 2018-2024, Sylabs Inc. All rights reserved.
 // This software is licensed under a 3-clause BSD license. Please consult the
 // LICENSE.md file distributed with the sources of this project regarding your
 // rights to use or distribute this software.
@@ -49,9 +49,9 @@ func TestMakeFiles(t *testing.T) {
 		if err := makeDirs(d); err != nil {
 			return err
 		}
-		return makeFiles(d)
+		return makeFiles(d, false)
 	})
-	testWithBadDir(t, makeFiles)
+	testWithBadDir(t, func(d string) error { return makeFiles(d, false) })
 	// #4532 - Check that we can succeed with an existing file that doesn't have
 	// write permission.
 	testWithGoodDir(t, func(d string) error {
@@ -62,14 +62,14 @@ func TestMakeFiles(t *testing.T) {
 		if err != nil {
 			t.Fatalf("Failed to make test hosts file: %s", err)
 		}
-		return makeFiles(d)
+		return makeFiles(d, false)
 	})
 }
 
 func TestMakeBaseEnv(t *testing.T) {
 	test.DropPrivilege(t)
 	defer test.ResetPrivilege(t)
 
-	testWithGoodDir(t, makeBaseEnv)
-	testWithBadDir(t, makeBaseEnv)
+	testWithGoodDir(t, func(d string) error { return makeBaseEnv(d, false) })
+	testWithBadDir(t, func(d string) error { return makeBaseEnv(d, false) })
 }

internal/pkg/build/sources/conveyorPacker_arch.go

@@ -1,4 +1,4 @@
-// Copyright (c) 2018-2022, Sylabs Inc. All rights reserved.
+// Copyright (c) 2018-2024, Sylabs Inc. All rights reserved.
 // This software is licensed under a 3-clause BSD license. Please consult the
 // LICENSE.md file distributed with the sources of this project regarding your
 // rights to use or distribute this software.
@@ -204,7 +204,7 @@ func (cp *ArchConveyorPacker) getPacConf(pacmanConfURL string) (pacConf string,
 }
 
 func (cp *ArchConveyorPacker) insertBaseEnv() (err error) {
-	if err = makeBaseEnv(cp.b.RootfsPath); err != nil {
+	if err = makeBaseEnv(cp.b.RootfsPath, true); err != nil {
 		return
 	}
 	return nil

internal/pkg/build/sources/conveyorPacker_busybox.go

@@ -121,7 +121,7 @@ func (c *BusyBoxConveyor) insertBusyBox(mirrorurl string) (busyBoxPath string, e
 }
 
 func (c *BusyBoxConveyor) insertBaseEnv() (err error) {
-	if err = makeBaseEnv(c.b.RootfsPath); err != nil {
+	if err = makeBaseEnv(c.b.RootfsPath, true); err != nil {
 		return
 	}
 	return nil

internal/pkg/build/sources/conveyorPacker_debootstrap.go

@@ -1,4 +1,4 @@
-// Copyright (c) 2018-2022, Sylabs Inc. All rights reserved.
+// Copyright (c) 2018-2024, Sylabs Inc. All rights reserved.
 // This software is licensed under a 3-clause BSD license. Please consult the
 // LICENSE.md file distributed with the sources of this project regarding your
 // rights to use or distribute this software.
@@ -267,7 +267,7 @@ func (cp *DebootstrapConveyorPacker) getRecipeHeaderInfo() (err error) {
 }
 
 func (cp *DebootstrapConveyorPacker) insertBaseEnv(b *types.Bundle) (err error) {
-	if err = makeBaseEnv(b.RootfsPath); err != nil {
+	if err = makeBaseEnv(b.RootfsPath, true); err != nil {
 		return
 	}
 	return nil

internal/pkg/build/sources/conveyorPacker_library.go

@@ -1,5 +1,5 @@
 // Copyright (c) 2020, Control Command Inc. All rights reserved.
-// Copyright (c) 2018-2023, Sylabs Inc. All rights reserved.
+// Copyright (c) 2018-2024, Sylabs Inc. All rights reserved.
 // This software is licensed under a 3-clause BSD license. Please consult the
 // LICENSE.md file distributed with the sources of this project regarding your
 // rights to use or distribute this software.
@@ -38,10 +38,6 @@ func (cp *LibraryConveyorPacker) Get(ctx context.Context, b *types.Bundle) (err
 	libraryURL := b.Opts.LibraryURL
 	authToken := b.Opts.LibraryAuthToken
 
-	if err = makeBaseEnv(cp.b.RootfsPath); err != nil {
-		return fmt.Errorf("while inserting base environment: %v", err)
-	}
-
 	// check for custom library from definition
 	customLib, ok := b.Recipe.Header["library"]
 	if ok {
@@ -82,7 +78,7 @@ func (cp *LibraryConveyorPacker) Get(ctx context.Context, b *types.Bundle) (err
 	}
 
 	// insert base metadata before unpacking fs
-	if err = makeBaseEnv(cp.b.RootfsPath); err != nil {
+	if err = makeBaseEnv(cp.b.RootfsPath, true); err != nil {
 		return fmt.Errorf("while inserting base environment: %v", err)
 	}
 

internal/pkg/build/sources/conveyorPacker_local.go

@@ -1,4 +1,4 @@
-// Copyright (c) 2018-2022, Sylabs Inc. All rights reserved.
+// Copyright (c) 2018-2024, Sylabs Inc. All rights reserved.
 // This software is licensed under a 3-clause BSD license. Please consult the
 // LICENSE.md file distributed with the sources of this project regarding your
 // rights to use or distribute this software.
@@ -112,9 +112,11 @@ func (cp *LocalConveyorPacker) Pack(ctx context.Context) (*types.Bundle, error)
 		return nil, fmt.Errorf("while unpacking local image: %v", err)
 	}
 
-	// insert base metadata AFTER unpacking fs to avoid conflicts with contained files/symlinks
+	// Insert base metadata after unpacking fs to avoid unsquashfs failure on
+	// existing files/symlink. Call makeBaseEnv with overwrite=false so we don't
+	// overwrite runscripts etc. that were extracted from the image.
 	sylog.Infof("Inserting Singularity configuration...")
-	if err = makeBaseEnv(b.RootfsPath); err != nil {
+	if err = makeBaseEnv(b.RootfsPath, false); err != nil {
 		return nil, fmt.Errorf("while inserting base environment: %v", err)
 	}
 

internal/pkg/build/sources/conveyorPacker_oci.go

@@ -247,7 +247,7 @@ func (cp *OCIConveyorPacker) unpackRootfs(ctx context.Context) error {
 }
 
 func (cp *OCIConveyorPacker) insertBaseEnv() (err error) {
-	if err = makeBaseEnv(cp.b.RootfsPath); err != nil {
+	if err = makeBaseEnv(cp.b.RootfsPath, true); err != nil {
 		sylog.Errorf("%v", err)
 	}
 	return

internal/pkg/build/sources/conveyorPacker_oras.go

@@ -1,4 +1,4 @@
-// Copyright (c) 2020, Sylabs Inc. All rights reserved.
+// Copyright (c) 2020-2024, Sylabs Inc. All rights reserved.
 // This software is licensed under a 3-clause BSD license. Please consult the
 // LICENSE.md file distributed with the sources of this project regarding your
 // rights to use or distribute this software.
@@ -45,7 +45,7 @@ func (cp *OrasConveyorPacker) Get(ctx context.Context, b *types.Bundle) (err err
 	}
 
 	// insert base metadata before unpacking fs
-	if err = makeBaseEnv(b.RootfsPath); err != nil {
+	if err = makeBaseEnv(b.RootfsPath, true); err != nil {
 		return fmt.Errorf("while inserting base environment: %v", err)
 	}
 

internal/pkg/build/sources/conveyorPacker_scratch.go

@@ -1,4 +1,4 @@
-// Copyright (c) 2018-2022, Sylabs Inc. All rights reserved.
+// Copyright (c) 2018-2024, Sylabs Inc. All rights reserved.
 // This software is licensed under a 3-clause BSD license. Please consult the
 // LICENSE.md file distributed with the sources of this project regarding your
 // rights to use or distribute this software.
@@ -47,7 +47,7 @@ func (cp *ScratchConveyorPacker) Pack(context.Context) (b *types.Bundle, err err
 }
 
 func (c *ScratchConveyor) insertBaseEnv() (err error) {
-	if err = makeBaseEnv(c.b.RootfsPath); err != nil {
+	if err = makeBaseEnv(c.b.RootfsPath, true); err != nil {
 		return
 	}
 	return nil

internal/pkg/build/sources/conveyorPacker_shub.go

@@ -1,4 +1,4 @@
-// Copyright (c) 2018-2022, Sylabs Inc. All rights reserved.
+// Copyright (c) 2018-2024, Sylabs Inc. All rights reserved.
 // This software is licensed under a 3-clause BSD license. Please consult the
 // LICENSE.md file distributed with the sources of this project regarding your
 // rights to use or distribute this software.
@@ -34,7 +34,7 @@ func (cp *ShubConveyorPacker) Get(ctx context.Context, b *types.Bundle) (err err
 	}
 
 	// insert base metadata before unpacking fs
-	if err = makeBaseEnv(cp.b.RootfsPath); err != nil {
+	if err = makeBaseEnv(cp.b.RootfsPath, true); err != nil {
 		return fmt.Errorf("while inserting base environment: %v", err)
 	}