Merge fixes from master to release-3.10 for rc.2

.circleci/config.yml

@@ -6,7 +6,7 @@ orbs:
 parameters:
   go-version:
     type: string
-    default: '1.18.1'
+    default: '1.18.2'
 
 executors:
   node:

.github/dependabot.yml

@@ -15,3 +15,8 @@ updates:
     schedule:
       interval: "daily"
     target-branch: release-3.9
+  - package-ecosystem: "gomod" # See documentation for possible values
+    directory: "/" # Location of package manifests
+    schedule:
+      interval: "daily"
+    target-branch: release-3.10

CHANGELOG.md

@@ -1,5 +1,19 @@
 # SingularityCE Changelog
 
+## Changes Since Last Release
+
+### New features / functionalities
+
+- Add support for `%files` section in remote builds, when a compatible remote is
+  used.
+
+### Bug Fixes
+
+- Correctly launch CleanupHost process only when needed in `--sif-fuse` flow.
+- Add specific error for unreadable image / overlay file.
+- Ensure cgroups device limits are default allow per past behavior.
+- Improve error message when remote build server does not support the `%files` section.
+
 ## 3.10.0-rc.1 \[2022-05-04\]
 
 This is the first release candidate for the upcoming SingularityCE 3.10 release.

INSTALL.md

@@ -56,7 +56,7 @@ _**NOTE:** if you are updating Go from a older version, make sure you remove
 `/usr/local/go` before reinstalling it._
 
 ```sh
-export VERSION=1.18.1 OS=linux ARCH=amd64  # change this as you need
+export VERSION=1.18.2 OS=linux ARCH=amd64  # change this as you need
 
 wget -O /tmp/go${VERSION}.${OS}-${ARCH}.tar.gz \
   https://dl.google.com/go/go${VERSION}.${OS}-${ARCH}.tar.gz

LICENSE_THIRD_PARTY.md

@@ -502,3 +502,11 @@ Are code from the conmon project, under the Apache License, Version 2.0.
    See the License for the specific language governing permissions and
    limitations under the License.
 ```
+
+## github.com/docker/cli
+
+The source files:
+
+* `internal/app/singularity/instance_linux.go`
+
+Contain code from the docker cli project, under the Apache License, Version 2.0.

RELEASE_PROCEDURE.md

@@ -28,13 +28,17 @@ When a new 3.Y.0 minor version of SingularityCE is issued the release process
 begins by branching, and then issuing a release candidate for broader testing.
 
 When a new 3.Y.Z patch release is issued, the branch will already be present,
-and steps 1-2 should be skipped.
+and steps 1-4 should be skipped.
 
 1. From a repository that is up-to-date with master, create a release branch
    e.g. `git checkout upstream/master -b release-3.8`.
 1. Push the release branch to GitHub via `git push upstream release-3.8`.
 1. Examine the GitHub branch protection rules, to extend them to the new release
    branch if needed.
+1. Update the `.github/dependabot.yml` configuration so that dependabot is
+   tracking the new stable release branch. Do not remove the previous stable
+   release branch from the configuration yet, as it should be monitored until
+   the final release of a new 3.Y.0 version.
 1. Modify the `README.md`, `INSTALL.md`, `CHANGELOG.md` via PR against the
    release branch, so that they reflect the version to be released.
 1. Apply an annotated tag via
@@ -81,7 +85,7 @@ covered by tests.
    history from the RC process etc. is captured on `master`.
 1. If the release is a new major/minor version, move the prior `release-3.x`
    branch to `vault/release-3.x`.
-1. If the release is a new major/minor version, update the
-   `.github/dependabot.yml` configuration so that dependabot is tracking the new
-   stable release branch.
+1. If the release is a new final major/minor version, update the
+   `.github/dependabot.yml` configuration to remove the prior stable release
+   branch.
 1. Start scheduling / setting up milestones etc. to track the next release!

cmd/internal/cli/actions_linux.go

@@ -791,6 +791,10 @@ func execStarter(cobraCmd *cobra.Command, image string, args []string, name stri
 		}
 	}
 
+	if SIFFUSE && !(UserNamespace || insideUserNs) {
+		sylog.Warningf("--sif-fuse is not supported without user namespace, ignoring.")
+	}
+
 	// setuid workflow set RLIMIT_STACK to its default value,
 	// get the original value to restore it before executing
 	// container process
@@ -835,6 +839,7 @@ func execStarter(cobraCmd *cobra.Command, image string, args []string, name stri
 			starter.WithStdout(stdout),
 			starter.WithStderr(stderr),
 			starter.LoadOverlayModule(loadOverlay),
+			starter.CleanupHost(engineConfig.GetImageFuse()),
 		)
 
 		if sylog.GetLevel() != 0 {
@@ -866,6 +871,7 @@ func execStarter(cobraCmd *cobra.Command, image string, args []string, name stri
 			cfg,
 			starter.UseSuid(useSuid),
 			starter.LoadOverlayModule(loadOverlay),
+			starter.CleanupHost(engineConfig.GetImageFuse()),
 		)
 		sylog.Fatalf("%s", err)
 	}

cmd/internal/cli/instance_linux.go

@@ -19,6 +19,7 @@ func init() {
 		cmdManager.RegisterSubCmd(instanceCmd, instanceStartCmd)
 		cmdManager.RegisterSubCmd(instanceCmd, instanceStopCmd)
 		cmdManager.RegisterSubCmd(instanceCmd, instanceListCmd)
+		cmdManager.RegisterSubCmd(instanceCmd, instanceStatsCmd)
 	})
 }
 

cmd/internal/cli/instance_stats_linux.go

@@ -0,0 +1,76 @@
+// Copyright (c) 2022, Vanessa Sochat. All rights reserved.
+// This software is licensed under a 3-clause BSD license. Please consult the
+// LICENSE.md file distributed with the sources of this project regarding your
+// rights to use or distribute this software.
+
+package cli
+
+import (
+	"os"
+
+	"github.com/spf13/cobra"
+	"github.com/sylabs/singularity/docs"
+	"github.com/sylabs/singularity/internal/app/singularity"
+	"github.com/sylabs/singularity/pkg/cmdline"
+	"github.com/sylabs/singularity/pkg/sylog"
+)
+
+// Basic Design
+// singularity instance stats <name>
+// singularity instance stats --json <name>
+
+func init() {
+	addCmdInit(func(cmdManager *cmdline.CommandManager) {
+		cmdManager.RegisterFlagForCmd(&instanceStatsUserFlag, instanceStatsCmd)
+		cmdManager.RegisterFlagForCmd(&instanceStatsJSONFlag, instanceStatsCmd)
+	})
+}
+
+// -u|--user
+var instanceStatsUser string
+
+var instanceStatsUserFlag = cmdline.Flag{
+	ID:           "instanceStatsUserFlag",
+	Value:        &instanceStatsUser,
+	DefaultValue: "",
+	Name:         "user",
+	ShortHand:    "u",
+	Usage:        "view stats for an instance belonging to a user (root only)",
+	Tag:          "<username>",
+	EnvKeys:      []string{"USER"},
+}
+
+// -j|--json
+var instanceStatsJSON bool
+
+var instanceStatsJSONFlag = cmdline.Flag{
+	ID:           "instanceStatsJSONFlag",
+	Value:        &instanceStatsJSON,
+	DefaultValue: false,
+	Name:         "json",
+	ShortHand:    "j",
+	Usage:        "output stats in json",
+}
+
+// singularity instance stats
+var instanceStatsCmd = &cobra.Command{
+	Args:                  cobra.ExactArgs(1),
+	DisableFlagsInUseLine: true,
+	RunE: func(cmd *cobra.Command, args []string) error {
+		uid := os.Getuid()
+
+		// Root is required to look at stats for another user
+		if instanceStatsUser != "" && uid != 0 {
+			sylog.Fatalf("Only the root user can look at stats of a user's instance")
+		}
+
+		// Instance name is the only arg
+		name := args[0]
+		return singularity.InstanceStats(name, instanceStatsUser, instanceStatsJSON)
+	},
+
+	Use:     docs.InstanceStatsUse,
+	Short:   docs.InstanceStatsShort,
+	Long:    docs.InstanceStatsLong,
+	Example: docs.InstanceStatsExample,
+}

cmd/starter/c/include/starter.h

@@ -161,9 +161,6 @@ struct starter {
 
     /* bounding capability set will include caps needed by nvidia-container-cli */
     bool nvCCLICaps;
-
-    /* is a CLEANUP_HOST process require outside of namespaces for SIF FUSE cleanup */
-    bool cleanupHost;
 };
 
 /* engine configuration */

cmd/starter/c/starter.c

@@ -1291,17 +1291,13 @@ __attribute__((constructor)) static void init(void) {
     /* retrieve engine configuration from environment variables */
     read_engine_config(&sconfig->engine);
 
-    /* cleanup environment variables */
-    cleanenv();
-
-    /* fix I/O streams to point to /dev/null if they are closed */
-    fix_streams();
-
-    /* set an invalid value for check */
-    sconfig->starter.workingDirectoryFd = -1;
-
     // Unpriv host cleanup in calling namespaces for SIF FUSE mount
-    if ( !sconfig->starter.cleanupHost ) {   
+    if ( getenv("CLEANUP_HOST") != NULL ) {
+        // FUSE SIF mount isn't supported in setuid flow at present.
+        // We should never have a CleanupHost process in setuid mode - enforce this.
+        if ( sconfig->starter.isSuid ) {
+           fatalf("CleanupHost process requested in setuid mode. Not permitted.\n");
+        }
         debugf("Create socketpair for cleanup communication channel\n");
             if ( socketpair(AF_UNIX, SOCK_STREAM|SOCK_CLOEXEC, 0, cleanup_socket) < 0 ) {
                 fatalf("Failed to create communication socket: %s\n", strerror(errno));
@@ -1315,12 +1311,23 @@ __attribute__((constructor)) static void init(void) {
             verbosef("Spawn CleanupHost\n");
             goexecute = CLEANUP_HOST;
             return;
-        }else{
+        } else {
             // In master - Close child end of cleanup socket
             close(cleanup_socket[1]);
         }
+    } else {
+        debugf("CleanupHost not requested\n");
     }
 
+    /* cleanup environment variables */
+    cleanenv();
+
+    /* fix I/O streams to point to /dev/null if they are closed */
+    fix_streams();
+
+    /* set an invalid value for check */
+    sconfig->starter.workingDirectoryFd = -1;
+
     /* save opened file descriptors that won't be closed when stage 1 exits */
     master_fds = list_fd();
 

docs/content.go

@@ -100,6 +100,9 @@ Enterprise Performance Computing (EPC)`
 
   DEFFILE SECTIONS:
 
+  The following sections are presented in the order of processing, with the exception
+  that labels and environment can also be manipulated in %post.
+
       %pre
           echo "This is a scriptlet that will be executed on the host, as root before"
           echo "the container has been bootstrapped. This section is not commonly used."
@@ -109,10 +112,20 @@ Enterprise Performance Computing (EPC)`
           echo "the container has been bootstrapped. To install things into the container"
           echo "reference the file system location with $SINGULARITY_ROOTFS."
 
+      %files
+          /path/on/host/file.txt /path/on/container/file.txt
+          relative_file.txt /path/on/container/relative_file.txt
+
       %post
           echo "This scriptlet section will be executed from within the container after"
           echo "the bootstrap/base has been created and setup."
 
+      %environment
+          LUKE=goodguy
+          VADER=badguy
+          HAN=someguy
+          export HAN VADER LUKE
+
       %test
           echo "Define any test commands that should be executed after container has been"
           echo "built. This scriptlet will be executed from within the running container"
@@ -131,16 +144,6 @@ Enterprise Performance Computing (EPC)`
           HELLO MOTO
           KEY VALUE
 
-      %files
-          /path/on/host/file.txt /path/on/container/file.txt
-          relative_file.txt /path/on/container/relative_file.txt
-
-      %environment
-          LUKE=goodguy
-          VADER=badguy
-          HAN=someguy
-          export HAN VADER LUKE
-
       %help
           This is a text file to be displayed with the run-help command.
 
@@ -551,6 +554,20 @@ Enterprise Performance Computing (EPC)`
   $ singularity instance stop /tmp/my-sql.sif mysql
   Stopping /tmp/my-sql.sif mysql`
 
+	// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+	// instance stats
+	// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+	InstanceStatsUse   string = `stats [stats options...] <instance name>`
+	InstanceStatsShort string = `Get stats for a named instance`
+	InstanceStatsLong  string = `
+  The instance stats command allows you to get statistics for a named instance,
+  either printed to the terminal or in json. If you are root, you can optionally
+  ask for statistics for a container instance belonging to a specific user.`
+	InstanceStatsExample string = `
+  $ singularity instance stats mysql
+  $ singularity instance stats --json mysql
+  $ sudo singularity instance stats --user <username> user-mysql`
+
 	// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 	// instance stop
 	// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

e2e/actions/actions.go

@@ -2352,6 +2352,30 @@ func (c actionTests) actionSIFFUSE(t *testing.T) {
 	}
 }
 
+// Verify that the FUSE mounts, and the CleanupHost() process are not seen when
+// --sif-fuse should not be in effect.
+func (c actionTests) actionNoSIFFUSE(t *testing.T) {
+	e2e.EnsureImage(t, c.env)
+
+	profiles := []e2e.Profile{e2e.UserProfile, e2e.RootProfile, e2e.FakerootProfile, e2e.UserNamespaceProfile}
+
+	for _, p := range profiles {
+		c.env.RunSingularity(
+			t,
+			e2e.AsSubtest(p.String()),
+			e2e.WithProfile(p),
+			e2e.WithCommand("exec"),
+			e2e.WithGlobalOptions("-d"),
+			e2e.WithArgs(c.env.ImagePath, "mount"),
+			e2e.ExpectExit(
+				0,
+				e2e.ExpectError(e2e.UnwantedContainMatch, "squashfuse"),
+				e2e.ExpectError(e2e.UnwantedContainMatch, "CleanupHost()"),
+			),
+		)
+	}
+}
+
 func countSquashfuseMounts(t *testing.T) int {
 	count := 0
 
@@ -2413,5 +2437,6 @@ func E2ETests(env e2e.TestEnv) testhelper.Tests {
 		"compat":                c.actionCompat,        // test --compat
 		"invalidRemote":         np(c.invalidRemote),   // GHSA-5mv9-q7fq-9394
 		"SIFFUSE":               np(c.actionSIFFUSE),   // test --sif-fuse
+		"NoSIFFUSE":             np(c.actionNoSIFFUSE), // test absence of squashfs and CleanupHost()
 	}
 }