pod and qos level cgroup support

cmd/kubelet/app/server.go

@@ -441,6 +441,7 @@ func run(s *options.KubeletServer, kubeDeps *kubelet.KubeletDeps) (err error) {
 			ContainerRuntime:       s.ContainerRuntime,
 			CgroupsPerQOS:          s.CgroupsPerQOS,
 			CgroupRoot:             s.CgroupRoot,
+			CgroupDriver:           s.CgroupDriver,
 			ProtectKernelDefaults:  s.ProtectKernelDefaults,
 			RuntimeIntegrationType: s.ExperimentalRuntimeIntegrationType,
 		})

docs/devel/e2e-node-tests.md

@@ -210,8 +210,6 @@ make test_e2e_node TEST_ARGS="--disable-kubenet=false" # disable kubenet
 
 For testing with the QoS Cgroup Hierarchy enabled, you can pass --cgroups-per-qos flag as an argument into Ginkgo using TEST_ARGS
 
-*Note: Disabled pending feature stabilization.*
-
 ```sh
 make test_e2e_node TEST_ARGS="--cgroups-per-qos=true"
 ```

hack/local-up-cluster.sh

@@ -29,6 +29,14 @@ NET_PLUGIN=${NET_PLUGIN:-""}
 NET_PLUGIN_DIR=${NET_PLUGIN_DIR:-""}
 KUBE_ROOT=$(dirname "${BASH_SOURCE}")/..
 SERVICE_CLUSTER_IP_RANGE=${SERVICE_CLUSTER_IP_RANGE:-10.0.0.0/24}
+# if enabled, must set CGROUP_ROOT
+CGROUPS_PER_QOS=${CGROUPS_PER_QOS:-false}
+# this is not defaulted to preserve backward compatibility.
+# if CGROUPS_PER_QOS is enabled, recommend setting to /
+CGROUP_ROOT=${CGROUP_ROOT:""}
+# name of the cgroup driver, i.e. cgroupfs or systemd
+CGROUP_DRIVER=${CGROUP_DRIVER:-""}
+
 # We disable cluster DNS by default because this script uses docker0 (or whatever
 # container bridge docker is currently using) and we don't know the IP of the
 # DNS pod to pass in as --cluster-dns. To set this up by hand, set this flag
@@ -464,6 +472,9 @@ function start_kubelet {
         --feature-gates="${FEATURE_GATES}" \
         --cpu-cfs-quota=${CPU_CFS_QUOTA} \
         --enable-controller-attach-detach="${ENABLE_CONTROLLER_ATTACH_DETACH}" \
+        --cgroups-per-qos=${CGROUPS_PER_QOS} \
+        --cgroup-driver=${CGROUP_DRIVER} \
+        --cgroup-root=${CGROUP_ROOT} \
         ${dns_args} \
         ${net_plugin_dir_args} \
         ${net_plugin_args} \

pkg/apis/componentconfig/v1alpha1/defaults.go

@@ -368,13 +368,23 @@ func SetDefaults_KubeletConfiguration(obj *KubeletConfiguration) {
 		temp := int32(defaultIPTablesDropBit)
 		obj.IPTablesDropBit = &temp
 	}
-	if obj.CgroupDriver == "" {
-		obj.CgroupDriver = "cgroupfs"
-	}
 	if obj.CgroupsPerQOS == nil {
 		temp := false
 		obj.CgroupsPerQOS = &temp
 	}
+	if obj.CgroupDriver == "" {
+		obj.CgroupDriver = "cgroupfs"
+	}
+	// NOTE: this is for backwards compatibility with earlier releases where cgroup-root was optional.
+	// if cgroups per qos is not enabled, and cgroup-root is not specified, we need to default to the
+	// container runtime default and not default to the root cgroup.
+	if obj.CgroupsPerQOS != nil {
+		if *obj.CgroupsPerQOS {
+			if obj.CgroupRoot == "" {
+				obj.CgroupRoot = "/"
+			}
+		}
+	}
 }
 
 func boolVar(b bool) *bool {

pkg/kubelet/cm/BUILD

@@ -28,6 +28,7 @@ go_library(
         "//pkg/api/resource:go_default_library",
         "//pkg/kubelet/cadvisor:go_default_library",
         "//pkg/kubelet/qos:go_default_library",
+        "//pkg/types:go_default_library",
         "//pkg/util:go_default_library",
         "//pkg/util/errors:go_default_library",
         "//pkg/util/mount:go_default_library",
@@ -41,16 +42,23 @@ go_library(
         "//vendor:github.com/golang/glog",
         "//vendor:github.com/opencontainers/runc/libcontainer/cgroups",
         "//vendor:github.com/opencontainers/runc/libcontainer/cgroups/fs",
+        "//vendor:github.com/opencontainers/runc/libcontainer/cgroups/systemd",
         "//vendor:github.com/opencontainers/runc/libcontainer/configs",
     ],
 )
 
 go_test(
     name = "go_default_test",
-    srcs = ["container_manager_linux_test.go"],
+    srcs = [
+        "cgroup_manager_linux_test.go",
+        "container_manager_linux_test.go",
+        "helpers_linux_test.go",
+    ],
     library = "go_default_library",
     tags = ["automanaged"],
     deps = [
+        "//pkg/api:go_default_library",
+        "//pkg/api/resource:go_default_library",
         "//pkg/util/mount:go_default_library",
         "//vendor:github.com/stretchr/testify/assert",
         "//vendor:github.com/stretchr/testify/require",