Support rootless Podman driver, take 2

Usage: `minikube --rootless start --driver=podman --container-runtime=(cri-o|containerd)`
Tested on Podman 4.0.2, Ubuntu 21.10.

Needs cgroup v2 (as in Rootless Docker): https://rootlesscontaine.rs/getting-started/common/cgroup2/
See also `site/content/en/docs/drivers/includes/podman_usage.inc`

Fix issue 8719
Fix issue 12460
Replace PR 12901

Changes from PR 12901: `--rootless` is now a global flag (for ease of supporting `minikube delete`)

Signed-off-by: Akihiro Suda <akihiro.suda.cz@hco.ntt.co.jp>

cmd/minikube/cmd/root.go

@@ -22,6 +22,7 @@ import (
 	"os"
 	"path/filepath"
 	"runtime"
+	"strconv"
 	"strings"
 	"time"
 
@@ -72,6 +73,12 @@ var RootCmd = &cobra.Command{
 			out.WarningT("User name '{{.username}}' is not valid", out.V{"username": userName})
 			exit.Message(reason.Usage, "User name must be 60 chars or less.")
 		}
+		// viper maps $MINIKUBE_ROOTLESS to --rootless automatically, but it does not do vice versa,
+		// so we map --rootless to $MINIKUBE_ROOTLESS expliclity here.
+		// $MINIKUBE_ROOTLESS is referred by KIC runner, which isn't aware of the *cobra.Command object.
+		if cmd.Flag(config.RootlessFlag).Changed {
+			os.Setenv(constants.MinikubeRootlessEnv, strconv.FormatBool(viper.GetBool(config.RootlessFlag)))
+		}
 	},
 }
 
@@ -206,6 +213,7 @@ func init() {
 	RootCmd.PersistentFlags().StringP(config.ProfileName, "p", constants.DefaultClusterName, `The name of the minikube VM being used. This can be set to allow having multiple instances of minikube independently.`)
 	RootCmd.PersistentFlags().StringP(configCmd.Bootstrapper, "b", "kubeadm", "The name of the cluster bootstrapper that will set up the Kubernetes cluster.")
 	RootCmd.PersistentFlags().String(config.UserFlag, "", "Specifies the user executing the operation. Useful for auditing operations executed by 3rd party tools. Defaults to the operating system username.")
+	RootCmd.PersistentFlags().Bool(config.RootlessFlag, false, "Force to use rootless driver (docker and podman driver only)")
 
 	groups := templates.CommandGroups{
 		{

cmd/minikube/cmd/start_flags.go

@@ -552,12 +552,22 @@ func generateNewConfigFromFlags(cmd *cobra.Command, k8sVersion string, rtime str
 			exit.Message(reason.Usage, "Ensure your {{.driver_name}} is running and is healthy.", out.V{"driver_name": driver.FullName(drvName)})
 		}
 		if si.Rootless {
+			out.Styled(style.Notice, "Using rootless {{.driver_name}} driver", out.V{"driver_name": driver.FullName(drvName)})
 			if cc.KubernetesConfig.ContainerRuntime == constants.Docker {
 				exit.Message(reason.Usage, "--container-runtime must be set to \"containerd\" or \"cri-o\" for rootless")
 			}
 			// KubeletInUserNamespace feature gate is essential for rootless driver.
 			// See https://kubernetes.io/docs/tasks/administer-cluster/kubelet-in-userns/
 			cc.KubernetesConfig.FeatureGates = addFeatureGate(cc.KubernetesConfig.FeatureGates, "KubeletInUserNamespace=true")
+		} else {
+			if oci.IsRootlessForced() {
+				if driver.IsDocker(drvName) {
+					exit.Message(reason.Usage, "Using rootless Docker driver was required, but the current Docker does not seem rootless. Try 'docker context use rootless' .")
+				} else {
+					exit.Message(reason.Usage, "Using rootless driver was required, but the current driver does not seem rootless")
+				}
+			}
+			out.Styled(style.Notice, "Using {{.driver_name}} driver with the root privilege", out.V{"driver_name": driver.FullName(drvName)})
 		}
 		if si.StorageDriver == "btrfs" {
 			klog.Info("auto-setting LocalStorageCapacityIsolation to false because using btrfs storage driver")

pkg/drivers/kic/oci/cli_runner.go

@@ -31,6 +31,7 @@ import (
 
 	"k8s.io/klog/v2"
 
+	"k8s.io/minikube/pkg/minikube/constants"
 	"k8s.io/minikube/pkg/minikube/out"
 	"k8s.io/minikube/pkg/minikube/style"
 )
@@ -72,10 +73,40 @@ func (rr RunResult) Output() string {
 	return sb.String()
 }
 
+// IsRootlessForced returns whether rootless mode is explicitly required.
+func IsRootlessForced() bool {
+	s := os.Getenv(constants.MinikubeRootlessEnv)
+	if s == "" {
+		return false
+	}
+	v, err := strconv.ParseBool(s)
+	if err != nil {
+		klog.ErrorS(err, "failed to parse", "env", constants.MinikubeRootlessEnv, "value", s)
+		return false
+	}
+	return v
+}
+
+type prefixCmdOptions struct {
+	sudoFlags []string
+}
+
+type PrefixCmdOption func(*prefixCmdOptions)
+
+func WithSudoFlags(ss ...string) PrefixCmdOption {
+	return func(o *prefixCmdOptions) {
+		o.sudoFlags = ss
+	}
+}
+
 // PrefixCmd adds any needed prefix (such as sudo) to the command
-func PrefixCmd(cmd *exec.Cmd) *exec.Cmd {
-	if cmd.Args[0] == Podman && runtime.GOOS == "linux" { // want sudo when not running podman-remote
-		cmdWithSudo := exec.Command("sudo", append([]string{"-n"}, cmd.Args...)...)
+func PrefixCmd(cmd *exec.Cmd, opt ...PrefixCmdOption) *exec.Cmd {
+	var o prefixCmdOptions
+	for _, f := range opt {
+		f(&o)
+	}
+	if cmd.Args[0] == Podman && runtime.GOOS == "linux" && !IsRootlessForced() { // want sudo when not running podman-remote
+		cmdWithSudo := exec.Command("sudo", append(append([]string{"-n"}, o.sudoFlags...), cmd.Args...)...)
 		cmdWithSudo.Env = cmd.Env
 		cmdWithSudo.Dir = cmd.Dir
 		cmdWithSudo.Stdin = cmd.Stdin

pkg/drivers/kic/oci/info.go

@@ -59,7 +59,7 @@ func CachedDaemonInfo(ociBin string) (SysInfo, error) {
 func DaemonInfo(ociBin string) (SysInfo, error) {
 	if ociBin == Podman {
 		p, err := podmanSystemInfo()
-		cachedSysInfo = &SysInfo{CPUs: p.Host.Cpus, TotalMemory: p.Host.MemTotal, OSType: p.Host.Os, Swarm: false, StorageDriver: p.Store.GraphDriverName}
+		cachedSysInfo = &SysInfo{CPUs: p.Host.Cpus, TotalMemory: p.Host.MemTotal, OSType: p.Host.Os, Swarm: false, Rootless: p.Host.Security.Rootless, StorageDriver: p.Store.GraphDriverName}
 		return *cachedSysInfo, err
 	}
 	d, err := dockerSystemInfo()
@@ -213,8 +213,10 @@ type podmanSysInfo struct {
 		Hostname    string `json:"hostname"`
 		Kernel      string `json:"kernel"`
 		Os          string `json:"os"`
-		Rootless    bool   `json:"rootless"`
-		Uptime      string `json:"uptime"`
+		Security    struct {
+			Rootless bool `json:"rootless"`
+		} `json:"security"`
+		Uptime string `json:"uptime"`
 	} `json:"host"`
 	Registries struct {
 		Search []string `json:"search"`

pkg/drivers/kic/oci/oci.go

@@ -314,7 +314,7 @@ func createContainer(ociBin string, image string, opts ...createOpt) error {
 
 	// to run nested container from privileged container in podman https://bugzilla.redhat.com/show_bug.cgi?id=1687713
 	// only add when running locally (linux), when running remotely it needs to be configured on server in libpod.conf
-	if ociBin == Podman && runtime.GOOS == "linux" {
+	if ociBin == Podman && runtime.GOOS == "linux" && !IsRootlessForced() {
 		args = append(args, "--cgroup-manager", "cgroupfs")
 	}
 
@@ -344,7 +344,7 @@ func StartContainer(ociBin string, container string) error {
 
 	// to run nested container from privileged container in podman https://bugzilla.redhat.com/show_bug.cgi?id=1687713
 	// only add when running locally (linux), when running remotely it needs to be configured on server in libpod.conf
-	if ociBin == Podman && runtime.GOOS == "linux" {
+	if ociBin == Podman && runtime.GOOS == "linux" && !IsRootlessForced() {
 		args = append(args, "--cgroup-manager", "cgroupfs")
 	}
 

pkg/minikube/config/config.go

@@ -44,6 +44,8 @@ const (
 	ProfileName = "profile"
 	// UserFlag is the key for the global user flag (ex. --user=user1)
 	UserFlag = "user"
+	// RootlessFlag is the key for the global rootless flag (ex. --rootless=true)
+	RootlessFlag = "rootless"
 	// AddonImages stores custom addon images config
 	AddonImages = "addon-images"
 	// AddonRegistries stores custom addon images config

pkg/minikube/constants/constants.go

@@ -99,6 +99,8 @@ const (
 	TestDiskUsedEnv = "MINIKUBE_TEST_STORAGE_CAPACITY"
 	// TestDiskAvailableEnv is used in integration tests for insufficient storage with 'minikube status' (in GiB)
 	TestDiskAvailableEnv = "MINIKUBE_TEST_AVAILABLE_STORAGE"
+	// MinikubeRootlessEnv is used to force Rootless Docker/Podman driver
+	MinikubeRootlessEnv = "MINIKUBE_ROOTLESS"
 
 	// scheduled stop constants
 

pkg/minikube/registry/drvs/podman/podman.go

@@ -112,7 +112,8 @@ func status() registry.State {
 	cmd := exec.CommandContext(ctx, oci.Podman, "version", "--format", "{{.Server.Version}}")
 	// Run with sudo on linux (local), otherwise podman-remote (as podman)
 	if runtime.GOOS == "linux" {
-		cmd = exec.CommandContext(ctx, "sudo", "-k", "-n", oci.Podman, "version", "--format", "{{.Version}}")
+		cmd = exec.CommandContext(ctx, oci.Podman, "version", "--format", "{{.Version}}")
+		cmd = oci.PrefixCmd(cmd, oci.WithSudoFlags("-k"))
 		cmd.Env = append(os.Environ(), "LANG=C", "LC_ALL=C") // sudo is localized
 	}
 	o, err := cmd.Output()
@@ -150,7 +151,7 @@ func status() registry.State {
 		newErr := fmt.Errorf(`%q %v: %s`, strings.Join(cmd.Args, " "), exitErr, stderr)
 
 		if strings.Contains(stderr, "a password is required") && runtime.GOOS == "linux" {
-			return registry.State{Error: newErr, Installed: true, Healthy: false, Fix: fmt.Sprintf("Add your user to the 'sudoers' file: '%s ALL=(ALL) NOPASSWD: %s'", username, podman), Doc: "https://podman.io"}
+			return registry.State{Error: newErr, Installed: true, Healthy: false, Fix: fmt.Sprintf("Add your user to the 'sudoers' file: '%s ALL=(ALL) NOPASSWD: %s' , or specify '--rootless' to enable rootless mode", username, podman), Doc: "https://podman.io"}
 		}
 
 		// Typical low-level errors from running podman-remote:

site/content/en/docs/drivers/docker.md

@@ -45,6 +45,9 @@ docker context use rootless
 minikube start --driver=docker --container-runtime=containerd
 ```
 
+Unlike Podman driver, it is not necessary to supply the `--rootless` flag to the `minikube` command.
+When the `--rootless` flag is explicitly specified but the current Docker host is not rootless, minikube fails.
+
 The `--container-runtime` flag must be set to "containerd" or "cri-o".
 {{% /tab %}}
 {{% /tabs %}}

site/content/en/docs/drivers/includes/podman_usage.inc

@@ -21,3 +21,18 @@ To make podman the default driver:
 ```shell
 minikube config set driver podman
 ```
+
+## Rootless Podman
+
+By default, minikube executes Podman with `sudo`.
+To use Podman without `sudo` (i.e., Rootless Podman), run `minikube` with `--rootless` flag:
+
+```shell
+minikube --rootless start --driver=podman --container-runtime=cri-o
+```
+
+```shell
+minikube --rootless delete
+```
+
+See the [Rootless Docker](https://minikube.sigs.k8s.io/docs/drivers/docker/#rootless-docker) section for the list of supported `--container-runtime` values and restrictions.