40 changes: 29 additions & 11 deletions internal/pkg/runtime/engine/apptainer/prepare_linux.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -150,7 +150,7 @@ func (e *EngineOperations) PrepareConfig(starterConfig *starter.Config) error {
if err := e.prepareContainerConfig(starterConfig); err != nil {
return err
}
if err := e.loadImages(starterConfig); err != nil {
if err := e.loadImages(starterConfig, userNS); err != nil {
return err
}
}
Expand Down Expand Up @@ -1131,12 +1131,12 @@ func (e *EngineOperations) setSessionLayer(img *image.Image) error {
return nil
}

func (e *EngineOperations) loadImages(starterConfig *starter.Config) error {
func (e *EngineOperations) loadImages(starterConfig *starter.Config, userNS bool) error {
images := make([]image.Image, 0)

// load rootfs image
writable := e.EngineConfig.GetWritableImage()
img, err := e.loadImage(e.EngineConfig.GetImage(), writable)
img, err := e.loadImage(e.EngineConfig.GetImage(), writable, userNS)
if err != nil {
return err
}
Expand Down Expand Up @@ -1236,7 +1236,13 @@ func (e *EngineOperations) loadImages(starterConfig *starter.Config) error {
return fmt.Errorf("while getting overlay partitions in %s: %s", img.Path, err)
}
for _, p := range overlays {
if img.Writable && p.Type == image.EXT3 {
if p.Type != image.EXT3 {
continue
}
if !userNS && !e.EngineConfig.File.AllowSetuidMountExtfs {
return fmt.Errorf("configuration disallows users from mounting SIF extfs partition in setuid mode, try --userns")
}
if img.Writable {
writableOverlayPath = img.Path
}
}
Expand All @@ -1252,7 +1258,7 @@ func (e *EngineOperations) loadImages(starterConfig *starter.Config) error {

switch e.EngineConfig.GetSessionLayer() {
case apptainerConfig.OverlayLayer:
overlayImages, err := e.loadOverlayImages(starterConfig, writableOverlayPath)
overlayImages, err := e.loadOverlayImages(starterConfig, writableOverlayPath, userNS)
if err != nil {
return fmt.Errorf("while loading overlay images: %s", err)
}
Expand All @@ -1264,7 +1270,7 @@ func (e *EngineOperations) loadImages(starterConfig *starter.Config) error {
}
}

bindImages, err := e.loadBindImages(starterConfig)
bindImages, err := e.loadBindImages(starterConfig, userNS)
if err != nil {
return fmt.Errorf("while loading data bind images: %s", err)
}
Expand All @@ -1276,7 +1282,7 @@ func (e *EngineOperations) loadImages(starterConfig *starter.Config) error {
}

// loadOverlayImages loads overlay images.
func (e *EngineOperations) loadOverlayImages(starterConfig *starter.Config, writableOverlayPath string) ([]image.Image, error) {
func (e *EngineOperations) loadOverlayImages(starterConfig *starter.Config, writableOverlayPath string, userNS bool) ([]image.Image, error) {
images := make([]image.Image, 0)

for _, overlayImg := range e.EngineConfig.GetOverlayImage() {
Expand All @@ -1289,7 +1295,7 @@ func (e *EngineOperations) loadOverlayImages(starterConfig *starter.Config, writ
}
}

img, err := e.loadImage(splitted[0], writableOverlay)
img, err := e.loadImage(splitted[0], writableOverlay, userNS)
if err != nil {
if !image.IsReadOnlyFilesytem(err) {
return nil, fmt.Errorf("failed to open overlay image %s: %s", splitted[0], err)
Expand Down Expand Up @@ -1325,7 +1331,7 @@ func (e *EngineOperations) loadOverlayImages(starterConfig *starter.Config, writ
}

// loadBindImages load data bind images.
func (e *EngineOperations) loadBindImages(starterConfig *starter.Config) ([]image.Image, error) {
func (e *EngineOperations) loadBindImages(starterConfig *starter.Config, userNS bool) ([]image.Image, error) {
images := make([]image.Image, 0)

binds := e.EngineConfig.GetBindPath()
Expand All @@ -1339,7 +1345,7 @@ func (e *EngineOperations) loadBindImages(starterConfig *starter.Config) ([]imag

sylog.Debugf("Loading data image %s", imagePath)

img, err := e.loadImage(imagePath, !binds[i].Readonly())
img, err := e.loadImage(imagePath, !binds[i].Readonly(), userNS)
if err != nil && !image.IsReadOnlyFilesytem(err) {
return nil, fmt.Errorf("failed to load data image %s: %s", imagePath, err)
}
Expand All @@ -1355,7 +1361,7 @@ func (e *EngineOperations) loadBindImages(starterConfig *starter.Config) ([]imag
return images, nil
}

func (e *EngineOperations) loadImage(path string, writable bool) (*image.Image, error) {
func (e *EngineOperations) loadImage(path string, writable bool, userNS bool) (*image.Image, error) {
const delSuffix = " (deleted)"

imgObject, imgErr := image.Init(path, writable)
Expand Down Expand Up @@ -1413,18 +1419,27 @@ func (e *EngineOperations) loadImage(path string, writable bool) (*image.Image,
if !e.EngineConfig.File.AllowContainerSquashfs {
return nil, fmt.Errorf("configuration disallows users from running squashFS containers")
}
if !userNS && !e.EngineConfig.File.AllowSetuidMountSquashfs {
return nil, fmt.Errorf("configuration disallows users from mounting squashFS in setuid mode, try --userns")
}
// Bare EXT3
case image.EXT3:
if !e.EngineConfig.File.AllowContainerExtfs {
return nil, fmt.Errorf("configuration disallows users from running extFS containers")
}
if !userNS && !e.EngineConfig.File.AllowSetuidMountExtfs {
return nil, fmt.Errorf("configuration disallows users from mounting extfs in setuid mode, try --userns")
}
// Bare sandbox directory
case image.SANDBOX:
if !e.EngineConfig.File.AllowContainerDir {
return nil, fmt.Errorf("configuration disallows users from running sandbox containers")
}
// SIF
case image.SIF:
if !userNS && !e.EngineConfig.File.AllowSetuidMountSquashfs {
return nil, fmt.Errorf("configuration disallows users from mounting SIF squashFS partition in setuid mode, try --userns")
}
// Check if SIF contains an encrypted rootfs partition.
// We don't support encryption for other partitions at present.
encrypted, err := imgObject.HasEncryptedRootFs()
Expand All @@ -1435,6 +1450,9 @@ func (e *EngineOperations) loadImage(path string, writable bool) (*image.Image,
if encrypted && !e.EngineConfig.File.AllowContainerEncrypted {
return nil, fmt.Errorf("configuration disallows users from running encrypted SIF containers")
}
if encrypted && !userNS && !e.EngineConfig.File.AllowSetuidMountEncrypted {
return nil, fmt.Errorf("configuration disallows users from mounting encrypted files in setuid mode")
}
// SIF without encryption - regardless of rootfs filesystem type
if !encrypted && !e.EngineConfig.File.AllowContainerSIF {
return nil, fmt.Errorf("configuration disallows users from running unencrypted SIF containers")
Expand Down
125 changes: 84 additions & 41 deletions pkg/util/apptainerconf/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,46 +76,49 @@ func SetBinaryPath(libexecDir string, nonSuid bool) {

// File describes the apptainer.conf file options
type File struct {
AllowSetuid bool `default:"yes" authorized:"yes,no" directive:"allow setuid"`
AllowPidNs bool `default:"yes" authorized:"yes,no" directive:"allow pid ns"`
ConfigPasswd bool `default:"yes" authorized:"yes,no" directive:"config passwd"`
ConfigGroup bool `default:"yes" authorized:"yes,no" directive:"config group"`
ConfigResolvConf bool `default:"yes" authorized:"yes,no" directive:"config resolv_conf"`
MountProc bool `default:"yes" authorized:"yes,no" directive:"mount proc"`
MountSys bool `default:"yes" authorized:"yes,no" directive:"mount sys"`
MountDevPts bool `default:"yes" authorized:"yes,no" directive:"mount devpts"`
MountHome bool `default:"yes" authorized:"yes,no" directive:"mount home"`
MountTmp bool `default:"yes" authorized:"yes,no" directive:"mount tmp"`
MountHostfs bool `default:"no" authorized:"yes,no" directive:"mount hostfs"`
UserBindControl bool `default:"yes" authorized:"yes,no" directive:"user bind control"`
EnableFusemount bool `default:"yes" authorized:"yes,no" directive:"enable fusemount"`
EnableUnderlay bool `default:"yes" authorized:"yes,no" directive:"enable underlay"`
MountSlave bool `default:"yes" authorized:"yes,no" directive:"mount slave"`
AllowContainerSIF bool `default:"yes" authorized:"yes,no" directive:"allow container sif"`
AllowContainerEncrypted bool `default:"yes" authorized:"yes,no" directive:"allow container encrypted"`
AllowContainerSquashfs bool `default:"yes" authorized:"yes,no" directive:"allow container squashfs"`
AllowContainerExtfs bool `default:"yes" authorized:"yes,no" directive:"allow container extfs"`
AllowContainerDir bool `default:"yes" authorized:"yes,no" directive:"allow container dir"`
AlwaysUseNv bool `default:"no" authorized:"yes,no" directive:"always use nv"`
UseNvCCLI bool `default:"no" authorized:"yes,no" directive:"use nvidia-container-cli"`
AlwaysUseRocm bool `default:"no" authorized:"yes,no" directive:"always use rocm"`
SharedLoopDevices bool `default:"no" authorized:"yes,no" directive:"shared loop devices"`
MaxLoopDevices uint `default:"256" directive:"max loop devices"`
SessiondirMaxSize uint `default:"16" directive:"sessiondir max size"`
MountDev string `default:"yes" authorized:"yes,no,minimal" directive:"mount dev"`
EnableOverlay string `default:"try" authorized:"yes,no,try,driver" directive:"enable overlay"`
BindPath []string `default:"/etc/localtime,/etc/hosts" directive:"bind path"`
LimitContainerOwners []string `directive:"limit container owners"`
LimitContainerGroups []string `directive:"limit container groups"`
LimitContainerPaths []string `directive:"limit container paths"`
AllowNetUsers []string `directive:"allow net users"`
AllowNetGroups []string `directive:"allow net groups"`
AllowNetNetworks []string `directive:"allow net networks"`
RootDefaultCapabilities string `default:"full" authorized:"full,file,no" directive:"root default capabilities"`
MemoryFSType string `default:"tmpfs" authorized:"tmpfs,ramfs" directive:"memory fs type"`
CniConfPath string `directive:"cni configuration path"`
CniPluginPath string `directive:"cni plugin path"`
BinaryPath string `default:"$PATH:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" directive:"binary path"`
AllowSetuid bool `default:"yes" authorized:"yes,no" directive:"allow setuid"`
AllowPidNs bool `default:"yes" authorized:"yes,no" directive:"allow pid ns"`
ConfigPasswd bool `default:"yes" authorized:"yes,no" directive:"config passwd"`
ConfigGroup bool `default:"yes" authorized:"yes,no" directive:"config group"`
ConfigResolvConf bool `default:"yes" authorized:"yes,no" directive:"config resolv_conf"`
MountProc bool `default:"yes" authorized:"yes,no" directive:"mount proc"`
MountSys bool `default:"yes" authorized:"yes,no" directive:"mount sys"`
MountDevPts bool `default:"yes" authorized:"yes,no" directive:"mount devpts"`
MountHome bool `default:"yes" authorized:"yes,no" directive:"mount home"`
MountTmp bool `default:"yes" authorized:"yes,no" directive:"mount tmp"`
MountHostfs bool `default:"no" authorized:"yes,no" directive:"mount hostfs"`
UserBindControl bool `default:"yes" authorized:"yes,no" directive:"user bind control"`
EnableFusemount bool `default:"yes" authorized:"yes,no" directive:"enable fusemount"`
EnableUnderlay bool `default:"yes" authorized:"yes,no" directive:"enable underlay"`
MountSlave bool `default:"yes" authorized:"yes,no" directive:"mount slave"`
AllowContainerSIF bool `default:"yes" authorized:"yes,no" directive:"allow container sif"`
AllowContainerEncrypted bool `default:"yes" authorized:"yes,no" directive:"allow container encrypted"`
AllowContainerSquashfs bool `default:"yes" authorized:"yes,no" directive:"allow container squashfs"`
AllowContainerExtfs bool `default:"yes" authorized:"yes,no" directive:"allow container extfs"`
AllowContainerDir bool `default:"yes" authorized:"yes,no" directive:"allow container dir"`
AllowSetuidMountEncrypted bool `default:"yes" authorized:"yes,no" directive:"allow setuid-mount encrypted"`
AllowSetuidMountSquashfs bool `default:"yes" authorized:"yes,no" directive:"allow setuid-mount squashfs"`
AllowSetuidMountExtfs bool `default:"no" authorized:"yes,no" directive:"allow setuid-mount extfs"`
AlwaysUseNv bool `default:"no" authorized:"yes,no" directive:"always use nv"`
UseNvCCLI bool `default:"no" authorized:"yes,no" directive:"use nvidia-container-cli"`
AlwaysUseRocm bool `default:"no" authorized:"yes,no" directive:"always use rocm"`
SharedLoopDevices bool `default:"no" authorized:"yes,no" directive:"shared loop devices"`
MaxLoopDevices uint `default:"256" directive:"max loop devices"`
SessiondirMaxSize uint `default:"16" directive:"sessiondir max size"`
MountDev string `default:"yes" authorized:"yes,no,minimal" directive:"mount dev"`
EnableOverlay string `default:"try" authorized:"yes,no,try,driver" directive:"enable overlay"`
BindPath []string `default:"/etc/localtime,/etc/hosts" directive:"bind path"`
LimitContainerOwners []string `directive:"limit container owners"`
LimitContainerGroups []string `directive:"limit container groups"`
LimitContainerPaths []string `directive:"limit container paths"`
AllowNetUsers []string `directive:"allow net users"`
AllowNetGroups []string `directive:"allow net groups"`
AllowNetNetworks []string `directive:"allow net networks"`
RootDefaultCapabilities string `default:"full" authorized:"full,file,no" directive:"root default capabilities"`
MemoryFSType string `default:"tmpfs" authorized:"tmpfs,ramfs" directive:"memory fs type"`
CniConfPath string `directive:"cni configuration path"`
CniPluginPath string `directive:"cni plugin path"`
BinaryPath string `default:"$PATH:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" directive:"binary path"`
// SuidBinaryPath is hidden; it is not referenced below, and overwritten
SuidBinaryPath string `directive:"suidbinary path"`
MksquashfsProcs uint `default:"0" directive:"mksquashfs procs"`
Expand All @@ -127,6 +130,18 @@ type File struct {
SystemdCgroups bool `default:"yes" authorized:"yes,no" directive:"systemd cgroups"`
}

// NOTE: if you think that we may want to change the default for any
// configuration parameter in the future, it is a good idea to conditionally
// insert a comment before the default setting when the setting is equal
// to the current default. That enables the defaults to get updated in
// a new release even if an administrator has changed one of the *other*
// settings. This gets around the problem of packagers such as rpm
// refusing to overwrite a configuration file if any change has been made.
// This technique is used for example in the "allow setuid-mount" options
// below. If a default is changed in a future release, both the default
// setting above and the expression for the conditional comment below need
// to change at the same time.

const TemplateAsset = `# APPTAINER.CONF
# This is the global configuration file for Apptainer. This file controls
# what the container is allowed to do on a particular host, and as a result
Expand Down Expand Up @@ -321,7 +336,9 @@ sessiondir max size = {{ .SessiondirMaxSize }}
# ALLOW CONTAINER ${TYPE}: [BOOL]
# DEFAULT: yes
# This feature limits what kind of containers that Apptainer will allow
# users to use (note this does not apply for root).
# users to use (note this does not apply for root). Note that some of the
# same operations can be limited in setuid mode by the ALLOW SETUID-MOUNT
# feature below; both types need to be "yes" to be allowed.
#
# Allow use of unencrypted SIF containers
allow container sif = {{ if eq .AllowContainerSIF true}}yes{{ else }}no{{ end }}
Expand All @@ -334,6 +351,32 @@ allow container squashfs = {{ if eq .AllowContainerSquashfs true }}yes{{ else }}
allow container extfs = {{ if eq .AllowContainerExtfs true }}yes{{ else }}no{{ end }}
allow container dir = {{ if eq .AllowContainerDir true }}yes{{ else }}no{{ end }}
# ALLOW SETUID-MOUNT ${TYPE}: [BOOL]
# DEFAULT: yes, except no for extfs
# This feature limits what types of mounts that Apptainer will allow
# unprivileged users to use in setuid mode. Normally these operations
# require the elevated privileges of setuid mode, although Apptainer
# has unprivileged alternatives for squashfs and extfs. Note that some of
# the same operations can also be limited by the ALLOW CONTAINER feature
# above; both types need to be "yes" to be allowed.
#
# Allow mounting of SIF encryption (using the kernel device-mapper) in
# setuid mode
{{ if eq .AllowSetuidMountEncrypted true}}# {{ end }}allow setuid-mount encrypted = {{ if eq .AllowSetuidMountEncrypted true}}yes{{ else }}no{{ end }}
#
# Allow mounting of squashfs filesystem types in setuid mode, both inside and
# outside of SIF files
{{ if eq .AllowSetuidMountSquashfs true}}# {{ end }}allow setuid-mount squashfs = {{ if eq .AllowSetuidMountSquashfs true}}yes{{ else }}no{{ end }}
#
# Allow mounting of extfs filesystem types in setuid mode, both inside and
# outside of SIF files. WARNING: this filesystem type frequently has relevant
# CVEs that that take a very long time for vendors to patch because they are
# not considered to be High severity since normally unprivileged users do
# not have write access to the raw filesystem data. This is why this option
# defaults to "no". Change it at your own risk and consider using the
# LIMIT CONTAINER features above if you do.
{{ if eq .AllowSetuidMountExtfs false}}# {{ end }}allow setuid-mount extfs = {{ if eq .AllowSetuidMountExtfs true}}yes{{ else }}no{{ end }}
# ALLOW NET USERS: [STRING]
# DEFAULT: NULL
# Allow specified root administered CNI network configurations to be used by the
Expand Down
4 changes: 4 additions & 0 deletions scripts/should-e2e-run
View file
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,10 @@ case "${TARGET_BRANCH}" in
require_e2e=true
;;

null)
# Failed to read api, could be private repo. Run tests.
require_e2e=true
;;
*)
# The branch is not master or release, skip e2e
require_e2e=false
Expand Down