Skip to content
Permalink
Browse files
feat(buildah): native OCI rootless mode; vfs storage driver; bugfixes
Done:

* Added native OCI rootless Buildah mode. Provides more isolation and security
  in comparison to chroot mode. Works only outside of containers.
* $WERF_BUILDAH_MODE instead of $WERF_CONTAINER_RUNTIME_BUILDAN for
  Buidah mode/isolation configuration.
* VFS storage driver now available in addition to OverlayFS for both native and docker-with-fuse
  Buildah modes. Configurable via $WERF_BUILDAH_STORAGE_DRIVER.
* Lots of bugfixes. Minor refactoring. Tests.

Work in progress:

Trying to get rid of a need in external configuration, external dependencies
and work environment preparations to run `werf` in Buildah mode, especially
when running `werf` in containers. As of now no /etc/containers/* configs
needed anymore, lots of configuration built-in, some autodetection/autoconfiguration
implemented.
  • Loading branch information
ilya-lesikov committed Dec 8, 2021
1 parent 606c505 commit 58e92a28b65f2492b5388582740bcb18f3e26068
@@ -7,25 +7,78 @@ import (
"strings"

"github.com/werf/werf/pkg/buildah"
"github.com/werf/werf/pkg/buildah/types"
"github.com/werf/werf/pkg/container_runtime"
"github.com/werf/werf/pkg/docker"
"github.com/werf/werf/pkg/util"
)

func ContainerRuntimeProcessStartupHook() (bool, error) {
buildahMode := GetContainerRuntimeBuildahMode()
buildahMode, _, err := GetBuildahMode()
if err != nil {
return false, fmt.Errorf("unable to determine buildah mode: %s", err)
}

switch {
case buildahMode != "":
return buildah.ProcessStartupHook(buildahMode)
case *buildahMode != buildah.ModeDisabled:
return buildah.ProcessStartupHook(*buildahMode)
case strings.HasPrefix(os.Args[0], "buildah-") || strings.HasPrefix(os.Args[0], "chrootuser-") || strings.HasPrefix(os.Args[0], "storage-"):
return buildah.ProcessStartupHook("native-rootless")
return buildah.ProcessStartupHook(buildah.ModeNativeRootless)
}

return false, nil
}

func GetContainerRuntimeBuildahMode() buildah.Mode {
return buildah.Mode(os.Getenv("WERF_CONTAINER_RUNTIME_BUILDAH"))
func GetBuildahMode() (*buildah.Mode, *types.Isolation, error) {
var (
mode buildah.Mode
isolation types.Isolation
)

modeRaw := os.Getenv("WERF_BUILDAH_MODE")
switch modeRaw {
case "native-rootless":
if isInContainer, err := util.IsInContainer(); err != nil {
return nil, nil, fmt.Errorf("unable to determine if is in container: %s", err)
} else if isInContainer {
return nil, nil, fmt.Errorf("native rootless mode is not available in containers: %s", err)
}
mode = buildah.ModeNativeRootless
isolation = types.IsolationOCIRootless
case "native-chroot":
mode = buildah.ModeNativeRootless
isolation = types.IsolationChroot
case "docker-with-fuse":
mode = buildah.ModeDockerWithFuse
isolation = types.IsolationChroot
case "default", "auto":
mode = buildah.ModeAuto
var err error
isolation, err = buildah.GetDefaultIsolation()
if err != nil {
return nil, nil, fmt.Errorf("unable to determine default isolation: %s", err)
}
case "docker", "":
mode = buildah.ModeDisabled
default:
return nil, nil, fmt.Errorf("unexpected mode specified: %s", modeRaw)
}

return &mode, &isolation, nil
}

func GetBuildahStorageDriver() (*buildah.StorageDriver, error) {
storageDriverRaw := os.Getenv("WERF_BUILDAH_STORAGE_DRIVER")
var storageDriver buildah.StorageDriver
switch storageDriverRaw {
case string(buildah.StorageDriverOverlay), string(buildah.StorageDriverVFS):
storageDriver = buildah.StorageDriver(storageDriverRaw)
case "default", "auto", "":
storageDriver = buildah.DefaultStorageDriver
default:
return nil, fmt.Errorf("unexpected driver specified: %s", storageDriverRaw)
}
return &storageDriver, nil
}

func wrapContainerRuntime(containerRuntime container_runtime.ContainerRuntime) container_runtime.ContainerRuntime {
@@ -36,9 +89,13 @@ func wrapContainerRuntime(containerRuntime container_runtime.ContainerRuntime) c
}

func InitProcessContainerRuntime(ctx context.Context, cmdData *CmdData) (container_runtime.ContainerRuntime, context.Context, error) {
buildahMode := GetContainerRuntimeBuildahMode()
if buildahMode != "" {
resolvedMode := buildah.ResolveMode(buildahMode)
buildahMode, buildahIsolation, err := GetBuildahMode()
if err != nil {
return nil, ctx, fmt.Errorf("unable to determine buildah mode: %s", err)
}

if *buildahMode != buildah.ModeDisabled {
resolvedMode := buildah.ResolveMode(*buildahMode)
if resolvedMode == buildah.ModeDockerWithFuse {
newCtx, err := InitProcessDocker(ctx, cmdData)
if err != nil {
@@ -47,10 +104,17 @@ func InitProcessContainerRuntime(ctx context.Context, cmdData *CmdData) (contain
ctx = newCtx
}

storageDriver, err := GetBuildahStorageDriver()
if err != nil {
return nil, ctx, fmt.Errorf("unable to determine buildah container runtime storage driver: %s", err)
}

insecure := *cmdData.InsecureRegistry || *cmdData.SkipTlsVerifyRegistry
b, err := buildah.NewBuildah(resolvedMode, buildah.BuildahOpts{
CommonBuildahOpts: buildah.CommonBuildahOpts{
Insecure: insecure,
Insecure: insecure,
Isolation: buildahIsolation,
StorageDriver: storageDriver,
},
})
if err != nil {
@@ -399,7 +399,7 @@ func (c *Conveyor) checkContainerRuntimeSupported(ctx context.Context) error {
Please select only dockerfile images or delete all non-dockerfile images from your werf.yaml.
Or disable buildah runtime by unsetting WERF_CONTAINER_RUNTIME_BUILDAH environment variable.`, strings.Join(nonDockerfileImages, ", "))
Or disable buildah runtime by unsetting WERF_BUILDAH_MODE environment variable.`, strings.Join(nonDockerfileImages, ", "))
}

return nil
@@ -7,28 +7,45 @@ import (
"os"
"path/filepath"

"github.com/werf/werf/pkg/buildah/types"
"github.com/werf/werf/pkg/util"
)

type BaseBuildah struct {
TmpDir string
Insecure bool
Isolation types.Isolation
TmpDir string
InstanceTmpDir string
SignaturePolicyPath string
Insecure bool
}

type BaseBuildahOpts struct {
Insecure bool
Isolation types.Isolation
Insecure bool
}

func NewBaseBuildah(tmpDir string, opts BaseBuildahOpts) (*BaseBuildah, error) {
b := &BaseBuildah{
TmpDir: tmpDir,
Insecure: opts.Insecure,
Isolation: opts.Isolation,
TmpDir: tmpDir,
Insecure: opts.Insecure,
}

if err := os.MkdirAll(b.TmpDir, os.ModePerm); err != nil {
return nil, fmt.Errorf("unable to create dir %q: %s", b.TmpDir, err)
}

var err error
b.InstanceTmpDir, err = ioutil.TempDir(b.TmpDir, "instance")
if err != nil {
return nil, fmt.Errorf("unable to create instance tmp dir: %s", err)
}

b.SignaturePolicyPath = filepath.Join(b.InstanceTmpDir, "policy.json")
if err := ioutil.WriteFile(b.SignaturePolicyPath, []byte(DefaultSignaturePolicy), os.ModePerm); err != nil {
return nil, fmt.Errorf("unable to write file %q: %s", b.SignaturePolicyPath, err)
}

return b, nil
}

@@ -5,19 +5,24 @@ import (
"fmt"
"io"
"os"
"os/exec"
"path/filepath"
"runtime"

"github.com/opencontainers/runtime-spec/specs-go"

"github.com/werf/werf/pkg/buildah/types"
"github.com/werf/werf/pkg/util"
"github.com/werf/werf/pkg/werf"
)

const (
DefaultShmSize = "65536k"
DefaultSignaturePolicy = `{"default": [{"type": "insecureAcceptAnything"}], "transports": {"docker-daemon": {"": [{"type": "insecureAcceptAnything"}]}}}`
BuildahImage = "ghcr.io/werf/buildah:v1.22.3-1"
BuildahStorageContainerName = "werf-buildah-storage"

DefaultStorageDriver StorageDriver = StorageDriverOverlay
)

type CommonOpts struct {
@@ -74,6 +79,7 @@ type Mode string

const (
ModeAuto Mode = "auto"
ModeDisabled Mode = "disabled"
ModeNativeRootless Mode = "native-rootless"
ModeDockerWithFuse Mode = "docker-with-fuse"
)
@@ -89,9 +95,18 @@ func ProcessStartupHook(mode Mode) (bool, error) {
}
}

type StorageDriver string

const (
StorageDriverOverlay StorageDriver = "overlay"
StorageDriverVFS StorageDriver = "vfs"
)

type CommonBuildahOpts struct {
TmpDir string
Insecure bool
Isolation *types.Isolation
StorageDriver *StorageDriver
TmpDir string
Insecure bool
}

type NativeRootlessModeOpts struct{}
@@ -105,6 +120,19 @@ type BuildahOpts struct {
}

func NewBuildah(mode Mode, opts BuildahOpts) (b Buildah, err error) {
if opts.CommonBuildahOpts.Isolation == nil {
defIsolation, err := GetDefaultIsolation()
if err != nil {
return b, fmt.Errorf("unable to determine default isolation: %s", err)
}
opts.CommonBuildahOpts.Isolation = &defIsolation
}

if opts.CommonBuildahOpts.StorageDriver == nil {
defStorageDriver := DefaultStorageDriver
opts.CommonBuildahOpts.StorageDriver = &defStorageDriver
}

if opts.CommonBuildahOpts.TmpDir == "" {
opts.CommonBuildahOpts.TmpDir = filepath.Join(werf.GetHomeDir(), "buildah", "tmp")
}
@@ -146,6 +174,33 @@ func ResolveMode(mode Mode) Mode {
}
}

func GetOverlayOptions() ([]string, error) {
fuseOverlayBinPath, err := exec.LookPath("fuse-overlayfs")
if err != nil {
return nil, fmt.Errorf("\"fuse-overlayfs\" binary not found in PATH: %s", err)
}

result := []string{fmt.Sprintf("overlay.mount_program=%s", fuseOverlayBinPath)}

if isInContainer, err := util.IsInContainer(); err != nil {
return nil, fmt.Errorf("unable to determine whether we are in the container: %s", err)
} else if isInContainer {
result = append(result, fmt.Sprintf("overlay.mountopt=%s", "nodev,fsync=0"))
}

return result, nil
}

func GetDefaultIsolation() (types.Isolation, error) {
if isInContainer, err := util.IsInContainer(); err != nil {
return 0, fmt.Errorf("unable to determine if is in container: %s", err)
} else if isInContainer {
return types.IsolationChroot, nil
} else {
return types.IsolationOCIRootless, nil
}
}

func debug() bool {
return os.Getenv("WERF_BUILDAH_DEBUG") == "1"
}

0 comments on commit 58e92a2

Please sign in to comment.