This commit contains the following:

1. Scheduler bug-fix + scheduler-focussed E2E tests
2. Add cgroup v2 support for in-place pod resize
3. Enable full E2E pod resize test for containerd>=1.6.9 and EventedPLEG related changes.

Co-Authored-By: Vinay Kulkarni <vskibum@gmail.com>

pkg/kubelet/cm/cgroup_manager_linux.go

@@ -32,6 +32,7 @@ import (
 	"github.com/opencontainers/runc/libcontainer/cgroups/manager"
 	cgroupsystemd "github.com/opencontainers/runc/libcontainer/cgroups/systemd"
 	libcontainerconfigs "github.com/opencontainers/runc/libcontainer/configs"
+	v1 "k8s.io/api/core/v1"
 	"k8s.io/klog/v2"
 	v1helper "k8s.io/kubernetes/pkg/apis/core/v1/helper"
 
@@ -47,7 +48,8 @@ const (
 	// MemoryMin is memory.min for cgroup v2
 	MemoryMin string = "memory.min"
 	// MemoryHigh is memory.high for cgroup v2
-	MemoryHigh string = "memory.high"
+	MemoryHigh         string = "memory.high"
+	Cgroup2MaxCpuLimit string = "max"
 )
 
 var RootCgroupName = CgroupName([]string{})
@@ -559,85 +561,188 @@ func (m *cgroupManagerImpl) MemoryUsage(name CgroupName) (int64, error) {
 	return int64(val), err
 }
 
-// Get the memory limit in bytes applied to the cgroup
-func (m *cgroupManagerImpl) GetCgroupMemoryConfig(name CgroupName) (uint64, error) {
-	cgroupPaths := m.buildCgroupPaths(name)
-	cgroupMemoryPath, found := cgroupPaths["memory"]
-	if !found {
-		return 0, fmt.Errorf("failed to build memory cgroup fs path for cgroup %v", name)
-	}
-	memLimit, err := fscommon.GetCgroupParamUint(cgroupMemoryPath, "memory.limit_in_bytes")
-	if err != nil {
-		return 0, fmt.Errorf("failed to get memory.limit_in_bytes for cgroup %v: %v", name, err)
-	}
-	return memLimit, nil
+// Convert cgroup v1 cpu.shares value to cgroup v2 cpu.weight
+// https://github.com/kubernetes/enhancements/tree/master/keps/sig-node/2254-cgroup-v2#phase-1-convert-from-cgroups-v1-settings-to-v2
+func CpuSharesToCpuWeight(cpuShares uint64) uint64 {
+	return uint64((((cpuShares - 2) * 9999) / 262142) + 1)
 }
 
-// Get the cpu quota, cpu period, and cpu shares applied to the cgroup
-func (m *cgroupManagerImpl) GetCgroupCpuConfig(name CgroupName) (int64, uint64, uint64, error) {
-	cgroupPaths := m.buildCgroupPaths(name)
-	cgroupCpuPath, found := cgroupPaths["cpu"]
-	if !found {
-		return 0, 0, 0, fmt.Errorf("failed to build CPU cgroup fs path for cgroup %v", name)
-	}
-	cpuQuotaStr, errQ := fscommon.GetCgroupParamString(cgroupCpuPath, "cpu.cfs_quota_us")
+// Convert cgroup v2 cpu.weight value to cgroup v1 cpu.shares
+// https://github.com/kubernetes/enhancements/tree/master/keps/sig-node/2254-cgroup-v2#phase-1-convert-from-cgroups-v1-settings-to-v2
+func CpuWeightToCpuShares(cpuWeight uint64) uint64 {
+	return uint64((((cpuWeight - 1) * 262142) / 9999) + 2)
+}
+
+func getCgroupv1CpuConfig(cgroupPath string) (*ResourceConfig, error) {
+	cpuQuotaStr, errQ := fscommon.GetCgroupParamString(cgroupPath, "cpu.cfs_quota_us")
 	if errQ != nil {
-		return 0, 0, 0, fmt.Errorf("failed to read CPU quota for cgroup %v: %v", name, errQ)
+		return nil, fmt.Errorf("failed to read CPU quota for cgroup %v: %v", cgroupPath, errQ)
 	}
 	cpuQuota, errInt := strconv.ParseInt(cpuQuotaStr, 10, 64)
 	if errInt != nil {
-		return 0, 0, 0, fmt.Errorf("failed to convert CPU quota as integer for cgroup %v: %v", name, errInt)
+		return nil, fmt.Errorf("failed to convert CPU quota as integer for cgroup %v: %v", cgroupPath, errInt)
 	}
-	cpuPeriod, errP := fscommon.GetCgroupParamUint(cgroupCpuPath, "cpu.cfs_period_us")
+	cpuPeriod, errP := fscommon.GetCgroupParamUint(cgroupPath, "cpu.cfs_period_us")
 	if errP != nil {
-		return 0, 0, 0, fmt.Errorf("failed to read CPU period for cgroup %v: %v", name, errP)
+		return nil, fmt.Errorf("failed to read CPU period for cgroup %v: %v", cgroupPath, errP)
 	}
-	cpuShares, errS := fscommon.GetCgroupParamUint(cgroupCpuPath, "cpu.shares")
-	if errP != nil {
-		return 0, 0, 0, fmt.Errorf("failed to read CPU shares for cgroup %v: %v", name, errS)
+	cpuShares, errS := fscommon.GetCgroupParamUint(cgroupPath, "cpu.shares")
+	if errS != nil {
+		return nil, fmt.Errorf("failed to read CPU shares for cgroup %v: %v", cgroupPath, errS)
+	}
+	return &ResourceConfig{CPUShares: &cpuShares, CPUQuota: &cpuQuota, CPUPeriod: &cpuPeriod}, nil
+}
+
+func getCgroupv2CpuConfig(cgroupPath string) (*ResourceConfig, error) {
+	var cpuLimitStr, cpuPeriodStr string
+	cpuLimitAndPeriod, err := fscommon.GetCgroupParamString(cgroupPath, "cpu.max")
+	if err != nil {
+		return nil, fmt.Errorf("failed to read cpu.max file for cgroup %v: %v", cgroupPath, err)
+	}
+	numItems, errScan := fmt.Sscanf(cpuLimitAndPeriod, "%s %s", &cpuLimitStr, &cpuPeriodStr)
+	if errScan != nil || numItems != 2 {
+		return nil, fmt.Errorf("failed to correctly parse content of cpu.max file ('%s') for cgroup %v: %v",
+			cpuLimitAndPeriod, cgroupPath, errScan)
+	}
+	cpuLimit := int64(-1)
+	if cpuLimitStr != Cgroup2MaxCpuLimit {
+		cpuLimit, err = strconv.ParseInt(cpuLimitStr, 10, 64)
+		if err != nil {
+			return nil, fmt.Errorf("failed to convert CPU limit as integer for cgroup %v: %v", cgroupPath, err)
+		}
+	}
+	cpuPeriod, errPeriod := strconv.ParseUint(cpuPeriodStr, 10, 64)
+	if errPeriod != nil {
+		return nil, fmt.Errorf("failed to convert CPU period as integer for cgroup %v: %v", cgroupPath, errPeriod)
+	}
+	cpuWeight, errWeight := fscommon.GetCgroupParamUint(cgroupPath, "cpu.weight")
+	if errWeight != nil {
+		return nil, fmt.Errorf("failed to read CPU weight for cgroup %v: %v", cgroupPath, errWeight)
+	}
+	cpuShares := CpuWeightToCpuShares(cpuWeight)
+	return &ResourceConfig{CPUShares: &cpuShares, CPUQuota: &cpuLimit, CPUPeriod: &cpuPeriod}, nil
+}
+
+func getCgroupCpuConfig(cgroupPath string) (*ResourceConfig, error) {
+	if libcontainercgroups.IsCgroup2UnifiedMode() {
+		return getCgroupv2CpuConfig(cgroupPath)
+	} else {
+		return getCgroupv1CpuConfig(cgroupPath)
 	}
-	return cpuQuota, cpuPeriod, cpuShares, nil
 }
 
-// Set the memory limit in bytes applied to the cgroup
-func (m *cgroupManagerImpl) SetCgroupMemoryConfig(name CgroupName, memoryLimit int64) error {
+func getCgroupMemoryConfig(cgroupPath string) (*ResourceConfig, error) {
+	memLimitFile := "memory.limit_in_bytes"
+	if libcontainercgroups.IsCgroup2UnifiedMode() {
+		memLimitFile = "memory.max"
+	}
+	memLimit, err := fscommon.GetCgroupParamUint(cgroupPath, memLimitFile)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read %s for cgroup %v: %v", memLimitFile, cgroupPath, err)
+	}
+	mLim := int64(memLimit)
+	//TODO(vinaykul,InPlacePodVerticalScaling): Add memory request support
+	return &ResourceConfig{Memory: &mLim}, nil
+
+}
+
+// Get the resource config values applied to the cgroup for specified resource type
+func (m *cgroupManagerImpl) GetCgroupConfig(name CgroupName, resource v1.ResourceName) (*ResourceConfig, error) {
 	cgroupPaths := m.buildCgroupPaths(name)
-	cgroupMemoryPath, found := cgroupPaths["memory"]
+	cgroupResourcePath, found := cgroupPaths[string(resource)]
 	if !found {
-		return fmt.Errorf("failed to build memory cgroup fs path for cgroup %v", name)
+		return nil, fmt.Errorf("failed to build %v cgroup fs path for cgroup %v", resource, name)
 	}
-	memLimit := strconv.FormatInt(memoryLimit, 10)
-	if err := os.WriteFile(filepath.Join(cgroupMemoryPath, "memory.limit_in_bytes"), []byte(memLimit), 0700); err != nil {
-		return fmt.Errorf("failed to write %v to %v: %v", memLimit, cgroupMemoryPath, err)
+	switch resource {
+	case v1.ResourceCPU:
+		return getCgroupCpuConfig(cgroupResourcePath)
+	case v1.ResourceMemory:
+		return getCgroupMemoryConfig(cgroupResourcePath)
 	}
-	return nil
+	return nil, fmt.Errorf("unsupported resource %v for cgroup %v", resource, name)
 }
 
-// Set the cpu quota, cpu period, and cpu shares applied to the cgroup
-func (m *cgroupManagerImpl) SetCgroupCpuConfig(name CgroupName, cpuQuota *int64, cpuPeriod, cpuShares *uint64) error {
+func setCgroupv1CpuConfig(cgroupPath string, resourceConfig *ResourceConfig) error {
 	var cpuQuotaStr, cpuPeriodStr, cpuSharesStr string
-	cgroupPaths := m.buildCgroupPaths(name)
-	cgroupCpuPath, found := cgroupPaths["cpu"]
-	if !found {
-		return fmt.Errorf("failed to build cpu cgroup fs path for cgroup %v", name)
+	if resourceConfig.CPUQuota != nil {
+		cpuQuotaStr = strconv.FormatInt(*resourceConfig.CPUQuota, 10)
+		if err := os.WriteFile(filepath.Join(cgroupPath, "cpu.cfs_quota_us"), []byte(cpuQuotaStr), 0700); err != nil {
+			return fmt.Errorf("failed to write %v to %v: %v", cpuQuotaStr, cgroupPath, err)
+		}
 	}
-	if cpuQuota != nil {
-		cpuQuotaStr = strconv.FormatInt(*cpuQuota, 10)
-		if err := os.WriteFile(filepath.Join(cgroupCpuPath, "cpu.cfs_quota_us"), []byte(cpuQuotaStr), 0700); err != nil {
-			return fmt.Errorf("failed to write %v to %v: %v", cpuQuotaStr, cgroupCpuPath, err)
+	if resourceConfig.CPUPeriod != nil {
+		cpuPeriodStr = strconv.FormatUint(*resourceConfig.CPUPeriod, 10)
+		if err := os.WriteFile(filepath.Join(cgroupPath, "cpu.cfs_period_us"), []byte(cpuPeriodStr), 0700); err != nil {
+			return fmt.Errorf("failed to write %v to %v: %v", cpuPeriodStr, cgroupPath, err)
 		}
 	}
-	if cpuPeriod != nil {
-		cpuPeriodStr = strconv.FormatUint(*cpuPeriod, 10)
-		if err := os.WriteFile(filepath.Join(cgroupCpuPath, "cpu.cfs_period_us"), []byte(cpuPeriodStr), 0700); err != nil {
-			return fmt.Errorf("failed to write %v to %v: %v", cpuPeriodStr, cgroupCpuPath, err)
+	if resourceConfig.CPUShares != nil {
+		cpuSharesStr = strconv.FormatUint(*resourceConfig.CPUShares, 10)
+		if err := os.WriteFile(filepath.Join(cgroupPath, "cpu.shares"), []byte(cpuSharesStr), 0700); err != nil {
+			return fmt.Errorf("failed to write %v to %v: %v", cpuSharesStr, cgroupPath, err)
 		}
 	}
-	if cpuShares != nil {
-		cpuSharesStr = strconv.FormatUint(*cpuShares, 10)
-		if err := os.WriteFile(filepath.Join(cgroupCpuPath, "cpu.shares"), []byte(cpuSharesStr), 0700); err != nil {
-			return fmt.Errorf("failed to write %v to %v: %v", cpuSharesStr, cgroupCpuPath, err)
+	return nil
+}
+
+func setCgroupv2CpuConfig(cgroupPath string, resourceConfig *ResourceConfig) error {
+	if resourceConfig.CPUQuota != nil {
+		if resourceConfig.CPUPeriod == nil {
+			return fmt.Errorf("CpuPeriod must be specified in order to set CpuLimit")
+		}
+		cpuLimitStr := Cgroup2MaxCpuLimit
+		if *resourceConfig.CPUQuota > -1 {
+			cpuLimitStr = strconv.FormatInt(*resourceConfig.CPUQuota, 10)
+		}
+		cpuPeriodStr := strconv.FormatUint(*resourceConfig.CPUPeriod, 10)
+		cpuMaxStr := fmt.Sprintf("%s %s", cpuLimitStr, cpuPeriodStr)
+		if err := os.WriteFile(filepath.Join(cgroupPath, "cpu.max"), []byte(cpuMaxStr), 0700); err != nil {
+			return fmt.Errorf("failed to write %v to %v: %v", cpuMaxStr, cgroupPath, err)
+		}
+	}
+	if resourceConfig.CPUShares != nil {
+		cpuWeight := CpuSharesToCpuWeight(*resourceConfig.CPUShares)
+		cpuWeightStr := strconv.FormatUint(cpuWeight, 10)
+		if err := os.WriteFile(filepath.Join(cgroupPath, "cpu.weight"), []byte(cpuWeightStr), 0700); err != nil {
+			return fmt.Errorf("failed to write %v to %v: %v", cpuWeightStr, cgroupPath, err)
 		}
 	}
 	return nil
 }
+
+func setCgroupCpuConfig(cgroupPath string, resourceConfig *ResourceConfig) error {
+	if libcontainercgroups.IsCgroup2UnifiedMode() {
+		return setCgroupv2CpuConfig(cgroupPath, resourceConfig)
+	} else {
+		return setCgroupv1CpuConfig(cgroupPath, resourceConfig)
+	}
+}
+
+func setCgroupMemoryConfig(cgroupPath string, resourceConfig *ResourceConfig) error {
+	memLimitFile := "memory.limit_in_bytes"
+	if libcontainercgroups.IsCgroup2UnifiedMode() {
+		memLimitFile = "memory.max"
+	}
+	memLimit := strconv.FormatInt(*resourceConfig.Memory, 10)
+	if err := os.WriteFile(filepath.Join(cgroupPath, memLimitFile), []byte(memLimit), 0700); err != nil {
+		return fmt.Errorf("failed to write %v to %v/%v: %v", memLimit, cgroupPath, memLimitFile, err)
+	}
+	//TODO(vinaykul,InPlacePodVerticalScaling): Add memory request support
+	return nil
+}
+
+// Set resource config for the specified resource type on the cgroup
+func (m *cgroupManagerImpl) SetCgroupConfig(name CgroupName, resource v1.ResourceName, resourceConfig *ResourceConfig) error {
+	cgroupPaths := m.buildCgroupPaths(name)
+	cgroupResourcePath, found := cgroupPaths[string(resource)]
+	if !found {
+		return fmt.Errorf("failed to build %v cgroup fs path for cgroup %v", resource, name)
+	}
+	switch resource {
+	case v1.ResourceCPU:
+		return setCgroupCpuConfig(cgroupResourcePath, resourceConfig)
+	case v1.ResourceMemory:
+		return setCgroupMemoryConfig(cgroupResourcePath, resourceConfig)
+	}
+	return nil
+}

pkg/kubelet/cm/cgroup_manager_linux_test.go

@@ -169,3 +169,85 @@ func TestParseSystemdToCgroupName(t *testing.T) {
 		}
 	}
 }
+
+func TestCpuSharesToCpuWeight(t *testing.T) {
+	testCases := []struct {
+		cpuShares         uint64
+		expectedCpuWeight uint64
+	}{
+		{
+			cpuShares:         2,
+			expectedCpuWeight: 1,
+		},
+		{
+			cpuShares:         3,
+			expectedCpuWeight: 1,
+		},
+		{
+			cpuShares:         4,
+			expectedCpuWeight: 1,
+		},
+		{
+			cpuShares:         28,
+			expectedCpuWeight: 1,
+		},
+		{
+			cpuShares:         29,
+			expectedCpuWeight: 2,
+		},
+		{
+			cpuShares:         245,
+			expectedCpuWeight: 10,
+		},
+		{
+			cpuShares:         262144,
+			expectedCpuWeight: 10000,
+		},
+	}
+
+	for _, testCase := range testCases {
+		if actual := CpuSharesToCpuWeight(testCase.cpuShares); actual != testCase.expectedCpuWeight {
+			t.Errorf("cpuShares: %v, expectedCpuWeight: %v, actualCpuWeight: %v",
+				testCase.cpuShares, testCase.expectedCpuWeight, actual)
+		}
+	}
+}
+
+func TestCpuWeightToCpuShares(t *testing.T) {
+	testCases := []struct {
+		cpuWeight         uint64
+		expectedCpuShares uint64
+	}{
+		{
+			cpuWeight:         1,
+			expectedCpuShares: 2,
+		},
+		{
+			cpuWeight:         2,
+			expectedCpuShares: 28,
+		},
+		{
+			cpuWeight:         3,
+			expectedCpuShares: 54,
+		},
+		{
+			cpuWeight:         4,
+			expectedCpuShares: 80,
+		},
+		{
+			cpuWeight:         245,
+			expectedCpuShares: 6398,
+		},
+		{
+			cpuWeight:         10000,
+			expectedCpuShares: 262144,
+		},
+	}
+
+	for _, testCase := range testCases {
+		if actual := CpuWeightToCpuShares(testCase.cpuWeight); actual != testCase.expectedCpuShares {
+			t.Errorf("cpuWeight: %v, expectedCpuShares: %v, actualCpuShares: %v",
+				testCase.cpuWeight, testCase.expectedCpuShares, actual)
+		}
+	}
+}

pkg/kubelet/cm/cgroup_manager_unsupported.go

@@ -19,7 +19,11 @@ limitations under the License.
 
 package cm
 
-import "errors"
+import (
+	"errors"
+
+	v1 "k8s.io/api/core/v1"
+)
 
 type unsupportedCgroupManager struct{}
 
@@ -77,19 +81,11 @@ func (m *unsupportedCgroupManager) ReduceCPULimits(cgroupName CgroupName) error
 	return nil
 }
 
-func (m *unsupportedCgroupManager) GetCgroupMemoryConfig(name CgroupName) (uint64, error) {
-	return 0, errNotSupported
-}
-
-func (m *unsupportedCgroupManager) GetCgroupCpuConfig(name CgroupName) (int64, uint64, uint64, error) {
-	return 0, 0, 0, errNotSupported
-}
-
-func (m *unsupportedCgroupManager) SetCgroupMemoryConfig(name CgroupName, memoryLimit int64) error {
-	return errNotSupported
+func (m *unsupportedCgroupManager) GetCgroupConfig(name CgroupName, resource v1.ResourceName) (*ResourceConfig, error) {
+	return nil, errNotSupported
 }
 
-func (m *unsupportedCgroupManager) SetCgroupCpuConfig(name CgroupName, cpuQuota *int64, cpuPeriod, cpuShares *uint64) error {
+func (m *unsupportedCgroupManager) SetCgroupConfig(name CgroupName, resource v1.ResourceName, resourceConfig *ResourceConfig) error {
 	return errNotSupported
 }