validation: test linux_cgroups_cpus with different values

Test CPU cgroups with various combinations of input values, as well as
an empty cgroup to check for the default values.
For that, create `validation/util/linux_resources_cpus.go` that holds
test helpers for CPU cgroups.

Determine number of CPUs correctly at runtime.
If possible, test also realtime runtime as well as realtime periods.
Print out result outputs in the TAP format.

Signed-off-by: Dongsu Park <dongsu@kinvolk.io>

generate/config.go

@@ -94,7 +94,8 @@ func (g *Generator) initConfigLinuxResourcesBlockIO() {
 	}
 }
 
-func (g *Generator) initConfigLinuxResourcesCPU() {
+// InitConfigLinuxResourcesCPU initializes CPU of Linux resources
+func (g *Generator) InitConfigLinuxResourcesCPU() {
 	g.initConfigLinuxResources()
 	if g.Config.Linux.Resources.CPU == nil {
 		g.Config.Linux.Resources.CPU = &rspec.LinuxCPU{}

generate/generate.go

@@ -703,43 +703,43 @@ func (g *Generator) DropLinuxResourcesBlockIOThrottleWriteIOPSDevice(major int64
 
 // SetLinuxResourcesCPUShares sets g.Config.Linux.Resources.CPU.Shares.
 func (g *Generator) SetLinuxResourcesCPUShares(shares uint64) {
-	g.initConfigLinuxResourcesCPU()
+	g.InitConfigLinuxResourcesCPU()
 	g.Config.Linux.Resources.CPU.Shares = &shares
 }
 
 // SetLinuxResourcesCPUQuota sets g.Config.Linux.Resources.CPU.Quota.
 func (g *Generator) SetLinuxResourcesCPUQuota(quota int64) {
-	g.initConfigLinuxResourcesCPU()
+	g.InitConfigLinuxResourcesCPU()
 	g.Config.Linux.Resources.CPU.Quota = &quota
 }
 
 // SetLinuxResourcesCPUPeriod sets g.Config.Linux.Resources.CPU.Period.
 func (g *Generator) SetLinuxResourcesCPUPeriod(period uint64) {
-	g.initConfigLinuxResourcesCPU()
+	g.InitConfigLinuxResourcesCPU()
 	g.Config.Linux.Resources.CPU.Period = &period
 }
 
 // SetLinuxResourcesCPURealtimeRuntime sets g.Config.Linux.Resources.CPU.RealtimeRuntime.
 func (g *Generator) SetLinuxResourcesCPURealtimeRuntime(time int64) {
-	g.initConfigLinuxResourcesCPU()
+	g.InitConfigLinuxResourcesCPU()
 	g.Config.Linux.Resources.CPU.RealtimeRuntime = &time
 }
 
 // SetLinuxResourcesCPURealtimePeriod sets g.Config.Linux.Resources.CPU.RealtimePeriod.
 func (g *Generator) SetLinuxResourcesCPURealtimePeriod(period uint64) {
-	g.initConfigLinuxResourcesCPU()
+	g.InitConfigLinuxResourcesCPU()
 	g.Config.Linux.Resources.CPU.RealtimePeriod = &period
 }
 
 // SetLinuxResourcesCPUCpus sets g.Config.Linux.Resources.CPU.Cpus.
 func (g *Generator) SetLinuxResourcesCPUCpus(cpus string) {
-	g.initConfigLinuxResourcesCPU()
+	g.InitConfigLinuxResourcesCPU()
 	g.Config.Linux.Resources.CPU.Cpus = cpus
 }
 
 // SetLinuxResourcesCPUMems sets g.Config.Linux.Resources.CPU.Mems.
 func (g *Generator) SetLinuxResourcesCPUMems(mems string) {
-	g.initConfigLinuxResourcesCPU()
+	g.InitConfigLinuxResourcesCPU()
 	g.Config.Linux.Resources.CPU.Mems = mems
 }
 

validation/linux_cgroups_cpus.go

@@ -2,68 +2,132 @@ package main
 
 import (
 	"fmt"
+	"os"
+	"path/filepath"
+	"runtime"
 
 	"github.com/mndrix/tap-go"
-	rspec "github.com/opencontainers/runtime-spec/specs-go"
 	"github.com/opencontainers/runtime-tools/cgroups"
 	"github.com/opencontainers/runtime-tools/validation/util"
 )
 
-func main() {
-	var shares uint64 = 1024
-	var period uint64 = 100000
-	var quota int64 = 50000
-	var cpus, mems string = "0-1", "0"
-	g, err := util.GetDefaultGenerator()
-	if err != nil {
-		util.Fatal(err)
+const (
+	defaultRealtimePeriod  uint64 = 1000000
+	defaultRealtimeRuntime int64  = 950000
+)
+
+func testCPUCgroups() error {
+	t := tap.New()
+	t.Header(0)
+	defer t.AutoPlan()
+
+	CPUrange := fmt.Sprintf("0-%d", runtime.NumCPU()-1)
+
+	// Test with different combinations of values.
+	// NOTE: most systems have only one memory node (mems=="0"), so we cannot
+	// simply test with multiple values of mems.
+	cases := []struct {
+		shares uint64
+		period uint64
+		quota  int64
+		cpus   string
+		mems   string
+	}{
+		{1024, 100000, 50000, "0", "0"},
+		{1024, 100000, 50000, CPUrange, "0"},
+		{1024, 100000, 200000, "0", "0"},
+		{1024, 100000, 200000, CPUrange, "0"},
+		{1024, 500000, 50000, "0", "0"},
+		{1024, 500000, 50000, CPUrange, "0"},
+		{1024, 500000, 200000, "0", "0"},
+		{1024, 500000, 200000, CPUrange, "0"},
+		{2048, 100000, 50000, "0", "0"},
+		{2048, 100000, 50000, CPUrange, "0"},
+		{2048, 100000, 200000, "0", "0"},
+		{2048, 100000, 200000, CPUrange, "0"},
+		{2048, 500000, 50000, "0", "0"},
+		{2048, 500000, 50000, CPUrange, "0"},
+		{2048, 500000, 200000, "0", "0"},
+		{2048, 500000, 200000, CPUrange, "0"},
 	}
-	g.SetLinuxCgroupsPath(cgroups.AbsCgroupPath)
-	g.SetLinuxResourcesCPUShares(shares)
-	g.SetLinuxResourcesCPUQuota(quota)
-	g.SetLinuxResourcesCPUPeriod(period)
-	g.SetLinuxResourcesCPUCpus(cpus)
-	g.SetLinuxResourcesCPUMems(mems)
-	err = util.RuntimeOutsideValidate(g, nil, func(config *rspec.Spec, t *tap.T, state *rspec.State) error {
-		cg, err := cgroups.FindCgroup()
-		if err != nil {
-			return err
-		}
-		lcd, err := cg.GetCPUData(state.Pid, config.Linux.CgroupsPath)
+
+	for _, c := range cases {
+		g, err := util.GetDefaultGenerator()
 		if err != nil {
-			return err
+			return fmt.Errorf("cannot get default config from generator: %v", err)
 		}
 
-		if lcd.Shares == nil {
-			return fmt.Errorf("unable to get cpu shares, lcd.Shares == %v", lcd.Shares)
+		g.SetLinuxCgroupsPath(cgroups.AbsCgroupPath)
+
+		if c.shares > 0 {
+			g.SetLinuxResourcesCPUShares(c.shares)
 		}
-		if *lcd.Shares != shares {
-			return fmt.Errorf("cpus shares limit is not set correctly, expect: %d, actual: %d", shares, *lcd.Shares)
+
+		if c.period > 0 {
+			g.SetLinuxResourcesCPUPeriod(c.period)
 		}
 
-		if lcd.Quota == nil {
-			return fmt.Errorf("unable to get cpu quota, lcd.Quota == %v", lcd.Quota)
+		if c.quota > 0 {
+			g.SetLinuxResourcesCPUQuota(c.quota)
 		}
-		if *lcd.Quota != quota {
-			return fmt.Errorf("cpus quota is not set correctly, expect: %d, actual: %d", quota, *lcd.Quota)
+
+		if c.cpus != "" {
+			g.SetLinuxResourcesCPUCpus(c.cpus)
 		}
 
-		if lcd.Period == nil {
-			return fmt.Errorf("unable to get cpu period, lcd.Period == %v", lcd.Period)
+		if c.mems != "" {
+			g.SetLinuxResourcesCPUMems(c.mems)
 		}
-		if *lcd.Period != period {
-			return fmt.Errorf("cpus period is not set correctly, expect: %d, actual: %d", period, *lcd.Period)
+
+		// NOTE: On most systems where CONFIG_RT_GROUP & CONFIG_RT_GROUP_SCHED are not enabled,
+		// the following tests will fail, because sysfs knobs like
+		// /sys/fs/cgroup/cpu,cpuacct/cpu.rt_{period,runtime}_us do not exist.
+		// So we need to check if the sysfs knobs exist before setting the variables.
+		if _, err := os.Stat(filepath.Join(util.CPUCgroupPrefix, "cpu.rt_period_us")); !os.IsNotExist(err) {
+			g.SetLinuxResourcesCPURealtimePeriod(defaultRealtimePeriod)
 		}
-		if lcd.Cpus != cpus {
-			return fmt.Errorf("cpus cpus is not set correctly, expect: %s, actual: %s", cpus, lcd.Cpus)
+
+		if _, err := os.Stat(filepath.Join(util.CPUCgroupPrefix, "cpu.rt_runtime_us")); !os.IsNotExist(err) {
+			g.SetLinuxResourcesCPURealtimeRuntime(defaultRealtimeRuntime)
 		}
-		if lcd.Mems != mems {
-			return fmt.Errorf("cpus mems is not set correctly, expect: %s, actual: %s", mems, lcd.Mems)
+
+		if err := util.RuntimeOutsideValidate(g, t, util.ValidateLinuxResourcesCPU); err != nil {
+			return fmt.Errorf("cannot validate CPU cgroups: %v", err)
 		}
-		return nil
-	})
+	}
+
+	return nil
+}
 
+func testEmptyCPU() error {
+	t := tap.New()
+	t.Header(0)
+	defer t.AutoPlan()
+
+	g, err := util.GetDefaultGenerator()
 	if err != nil {
+		return fmt.Errorf("cannot get default config from generator: %v", err)
+	}
+	g.InitConfigLinuxResourcesCPU()
+	g.SetLinuxCgroupsPath(cgroups.AbsCgroupPath)
+
+	if err := util.RuntimeOutsideValidate(g, t, util.ValidateLinuxResourcesCPUEmpty); err != nil {
+		return fmt.Errorf("cannot validate empty CPU cgroups: %v", err)
+	}
+
+	return nil
+}
+
+func main() {
+	if "linux" != runtime.GOOS {
+		util.Fatal(fmt.Errorf("linux-specific cgroup test"))
+	}
+
+	if err := testCPUCgroups(); err != nil {
+		util.Fatal(err)
+	}
+
+	if err := testEmptyCPU(); err != nil {
 		util.Fatal(err)
 	}
 }

validation/util/linux_resources_cpus.go

@@ -0,0 +1,137 @@
+package util
+
+import (
+	"fmt"
+	"io/ioutil"
+	"path/filepath"
+	"runtime"
+	"strconv"
+	"strings"
+
+	"github.com/mndrix/tap-go"
+	rspec "github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/opencontainers/runtime-tools/cgroups"
+)
+
+const (
+	// CPUCgroupPrefix is default path prefix where CPU cgroups are created
+	CPUCgroupPrefix string = "/sys/fs/cgroup/cpu,cpuacct"
+)
+
+// ValidateLinuxResourcesCPU validates if Linux.Resources.CPU is set to
+// correct values, the same as given values in the config.
+func ValidateLinuxResourcesCPU(config *rspec.Spec, t *tap.T, state *rspec.State) error {
+	cg, err := cgroups.FindCgroup()
+	t.Ok((err == nil), "find cpu cgroup")
+	if err != nil {
+		t.Diagnostic(err.Error())
+		return nil
+	}
+
+	lcd, err := cg.GetCPUData(state.Pid, config.Linux.CgroupsPath)
+	t.Ok((err == nil), "get cpu cgroup data")
+	if err != nil {
+		t.Diagnostic(err.Error())
+		return nil
+	}
+
+	if lcd.Shares == nil || config.Linux.Resources.CPU.Shares == nil {
+		t.Diagnostic(fmt.Sprintf("unable to get cpu shares, lcd.Shares == %v, config.Linux.Resources.CPU.Shares == %v", lcd.Shares, config.Linux.Resources.CPU.Shares))
+		return nil
+	}
+	t.Ok(*lcd.Shares == *config.Linux.Resources.CPU.Shares, "cpu shares is set correctly")
+	t.Diagnosticf("expect: %d, actual: %d", *config.Linux.Resources.CPU.Shares, *lcd.Shares)
+
+	if lcd.Period == nil || config.Linux.Resources.CPU.Period == nil {
+		t.Diagnostic(fmt.Sprintf("unable to get cpu period, lcd.Period == %v, config.Linux.Resources.CPU.Period == %v", lcd.Period, config.Linux.Resources.CPU.Period))
+		return nil
+	}
+	t.Ok(*lcd.Period == *config.Linux.Resources.CPU.Period, "cpu period is set correctly")
+	t.Diagnosticf("expect: %d, actual: %d", *config.Linux.Resources.CPU.Period, *lcd.Period)
+
+	if lcd.Quota == nil || config.Linux.Resources.CPU.Quota == nil {
+		t.Diagnostic(fmt.Sprintf("unable to get cpu quota, lcd.Quota == %v, config.Linux.Resources.CPU.Quota == %v", lcd.Quota, config.Linux.Resources.CPU.Quota))
+		return nil
+	}
+	t.Ok(*lcd.Quota == *config.Linux.Resources.CPU.Quota, "cpu quota is set correctly")
+	t.Diagnosticf("expect: %d, actual: %d", *config.Linux.Resources.CPU.Quota, *lcd.Quota)
+
+	t.Ok(lcd.Cpus == config.Linux.Resources.CPU.Cpus, "cpu cpus is set correctly")
+	t.Diagnosticf("expect: %s, actual: %s", config.Linux.Resources.CPU.Cpus, lcd.Cpus)
+
+	t.Ok(lcd.Mems == config.Linux.Resources.CPU.Mems, "cpu mems is set correctly")
+	t.Diagnosticf("expect: %s, actual: %s", config.Linux.Resources.CPU.Mems, lcd.Mems)
+
+	return nil
+}
+
+// ValidateLinuxResourcesCPUEmpty validates Linux.Resources.CPU is set to
+// correct values, when each value are set to the default ones.
+func ValidateLinuxResourcesCPUEmpty(config *rspec.Spec, t *tap.T, state *rspec.State) error {
+	outShares, err := ioutil.ReadFile(filepath.Join(CPUCgroupPrefix, "cpu.shares"))
+	if err != nil {
+		return nil
+	}
+	sh, _ := strconv.Atoi(strings.TrimSpace(string(outShares)))
+	defaultShares := uint64(sh)
+
+	outPeriod, err := ioutil.ReadFile(filepath.Join(CPUCgroupPrefix, "cpu.cfs_period_us"))
+	if err != nil {
+		return nil
+	}
+	pe, _ := strconv.Atoi(strings.TrimSpace(string(outPeriod)))
+	defaultPeriod := uint64(pe)
+
+	outQuota, err := ioutil.ReadFile(filepath.Join(CPUCgroupPrefix, "cpu.cfs_quota_us"))
+	if err != nil {
+		return nil
+	}
+	qu, _ := strconv.Atoi(strings.TrimSpace(string(outQuota)))
+	defaultQuota := int64(qu)
+
+	defaultCpus := fmt.Sprintf("0-%d", runtime.NumCPU()-1)
+	defaultMems := "0"
+
+	cg, err := cgroups.FindCgroup()
+	t.Ok((err == nil), "find cpu cgroup")
+	if err != nil {
+		t.Diagnostic(err.Error())
+		return nil
+	}
+
+	lcd, err := cg.GetCPUData(state.Pid, config.Linux.CgroupsPath)
+	t.Ok((err == nil), "get cpu cgroup data")
+	if err != nil {
+		t.Diagnostic(err.Error())
+		return nil
+	}
+
+	if lcd.Shares == nil {
+		t.Diagnostic(fmt.Sprintf("unable to get cpu shares, lcd.Shares == %v", lcd.Shares))
+		return nil
+	}
+	t.Ok(*lcd.Shares == defaultShares, "cpu shares is set correctly")
+	t.Diagnosticf("expect: %d, actual: %d", defaultShares, *lcd.Shares)
+
+	if lcd.Period == nil {
+		t.Diagnostic(fmt.Sprintf("unable to get cpu period, lcd.Period == %v", lcd.Period))
+		return nil
+	}
+	t.Ok(*lcd.Period == defaultPeriod, "cpu period is set correctly")
+	t.Diagnosticf("expect: %d, actual: %d", defaultPeriod, *lcd.Period)
+
+	if lcd.Quota == nil {
+		t.Diagnostic(fmt.Sprintf("unable to get cpu quota, lcd.Quota == %v", lcd.Quota))
+		return nil
+	}
+	t.Ok(*lcd.Quota == defaultQuota, "cpu quota is set correctly")
+	t.Diagnosticf("expect: %d, actual: %d", defaultQuota, *lcd.Quota)
+
+	t.Ok(lcd.Cpus == defaultCpus, "cpu cpus is set correctly")
+	t.Diagnosticf("expect: %s, actual: %s", defaultCpus, lcd.Cpus)
+
+	t.Ok(lcd.Mems == defaultMems, "cpu mems is set correctly")
+	t.Diagnosticf("expect: %s, actual: %s", defaultMems, lcd.Mems)
+
+	return nil
+}