workaround Permission denied problem with overlayfs and FUSE lower layer

Signed-off-by: Dave Dykstra <2129743+DrDaveD@users.noreply.github.com>

CHANGELOG.md

@@ -22,6 +22,11 @@ Changes since v1.3.0-rc.1
   `preferred` value on the `enable underlay` configuration option.
   Also the `--underlay` option can be used in setuid mode or as the root
   user, although it was ignored previously.
+- Prefer again to use kernel overlayfs over fuse-overlayfs when a lower
+  layer is FUSE and there's no writable upper layer, undoing the change
+  from 1.2.0.  Another workaround was found for the problem that change
+  addressed.  This applies in both setuid mode and in user namespace
+  mode (except the latter not on CentOS7 where it isn't supported).
 - Fix `--sharens` failure on EL8.
 
 ## v1.3.0-rc.1 - \[2024-01-10\]

e2e/actions/actions.go

@@ -1965,7 +1965,7 @@ func (c actionTests) actionLayerType(t *testing.T) {
 			args: []string{
 				c.env.ImagePath,
 			},
-			expectType: "fuse",
+			expectType: "overlay",
 		},
 		{
 			name:    "UserWithUnderlayOption",
@@ -1999,7 +1999,7 @@ func (c actionTests) actionLayerType(t *testing.T) {
 			args: []string{
 				c.env.ImagePath,
 			},
-			expectType: "fuse",
+			expectType: "overlay",
 		},
 		{
 			name:    "UserNamespaceWithUnderlayOption",

internal/pkg/runtime/engine/apptainer/container_linux.go

@@ -875,6 +875,22 @@ mount:
 			return nil
 		}
 		return fmt.Errorf("could not mount %s: %s", mnt.Source, err)
+	} else if mnt.Type == "overlay" {
+		// The overlay mount succeeded, but under some conditions the
+		// kernel overlayfs exhibits a bizarre behavior where it returns
+		// Permission denied to the user unless the mount happens twice
+		// and in between the mountpoint is accessed.
+		// See https://github.com/apptainer/apptainer/issues/1945
+		if _, err = os.Stat(dest); err != nil {
+			return fmt.Errorf("while getting stat for %s: %s", dest, err)
+		}
+		sylog.Debugf("Unmounting and remounting overlay")
+		if err = c.rpcOps.Unmount(dest, 0); err != nil {
+			return fmt.Errorf("while unmounting %s: %s", dest, err)
+		}
+		if err = c.rpcOps.Mount(source, dest, mnt.Type, flags, optsString); err != nil {
+			return fmt.Errorf("while remounting %s: %s", dest, err)
+		}
 	}
 
 	return nil

internal/pkg/runtime/engine/apptainer/rpc/args.go

@@ -43,6 +43,12 @@ type MountArgs struct {
 	Data       string
 }
 
+// UnmountArgs defines the arguments to unmount.
+type UnmountArgs struct {
+	Target       string
+	Unmountflags int
+}
+
 // CryptArgs defines the arguments to mount.
 type CryptArgs struct {
 	Offset    uint64

internal/pkg/runtime/engine/apptainer/rpc/client/client.go

@@ -45,6 +45,24 @@ func (t *RPC) Mount(source string, target string, filesystem string, flags uintp
 	return err
 }
 
+// Unmount calls the unmount RPC using the supplied arguments.
+func (t *RPC) Unmount(target string, flags int) error {
+	arguments := &args.UnmountArgs{
+		Target:       target,
+		Unmountflags: flags,
+	}
+
+	var unmountErr error
+
+	err := t.Client.Call(t.Name+".Unmount", arguments, &unmountErr)
+	// RPC communication will take precedence over unmount error
+	if err == nil {
+		err = unmountErr
+	}
+
+	return err
+}
+
 // Decrypt calls the DeCrypt RPC using the supplied arguments.
 func (t *RPC) Decrypt(offset uint64, path string, key []byte, masterPid int) (string, error) {
 	arguments := &args.CryptArgs{

internal/pkg/runtime/engine/apptainer/rpc/server/server_linux.go

@@ -73,6 +73,14 @@ func (t *Methods) Mount(arguments *args.MountArgs, mountErr *error) (err error)
 	return
 }
 
+// Unmount performs an unmount with the specified arguments.
+func (t *Methods) Unmount(arguments *args.UnmountArgs, unmountErr *error) (err error) {
+	mainthread.Execute(func() {
+		*unmountErr = syscall.Unmount(arguments.Target, arguments.Unmountflags)
+	})
+	return
+}
+
 // Decrypt decrypts the loop device.
 func (t *Methods) Decrypt(arguments *args.CryptArgs, reply *string) (err error) {
 	cryptName := ""