diff --git a/generate/config.go b/generate/config.go index 9421b6d8f..164fdf141 100644 --- a/generate/config.go +++ b/generate/config.go @@ -94,7 +94,8 @@ func (g *Generator) initConfigLinuxResourcesBlockIO() { } } -func (g *Generator) initConfigLinuxResourcesCPU() { +// InitConfigLinuxResourcesCPU initializes CPU of Linux resources +func (g *Generator) InitConfigLinuxResourcesCPU() { g.initConfigLinuxResources() if g.Config.Linux.Resources.CPU == nil { g.Config.Linux.Resources.CPU = &rspec.LinuxCPU{} diff --git a/generate/generate.go b/generate/generate.go index 07b57b44b..900278f9f 100644 --- a/generate/generate.go +++ b/generate/generate.go @@ -703,43 +703,43 @@ func (g *Generator) DropLinuxResourcesBlockIOThrottleWriteIOPSDevice(major int64 // SetLinuxResourcesCPUShares sets g.Config.Linux.Resources.CPU.Shares. func (g *Generator) SetLinuxResourcesCPUShares(shares uint64) { - g.initConfigLinuxResourcesCPU() + g.InitConfigLinuxResourcesCPU() g.Config.Linux.Resources.CPU.Shares = &shares } // SetLinuxResourcesCPUQuota sets g.Config.Linux.Resources.CPU.Quota. func (g *Generator) SetLinuxResourcesCPUQuota(quota int64) { - g.initConfigLinuxResourcesCPU() + g.InitConfigLinuxResourcesCPU() g.Config.Linux.Resources.CPU.Quota = "a } // SetLinuxResourcesCPUPeriod sets g.Config.Linux.Resources.CPU.Period. func (g *Generator) SetLinuxResourcesCPUPeriod(period uint64) { - g.initConfigLinuxResourcesCPU() + g.InitConfigLinuxResourcesCPU() g.Config.Linux.Resources.CPU.Period = &period } // SetLinuxResourcesCPURealtimeRuntime sets g.Config.Linux.Resources.CPU.RealtimeRuntime. func (g *Generator) SetLinuxResourcesCPURealtimeRuntime(time int64) { - g.initConfigLinuxResourcesCPU() + g.InitConfigLinuxResourcesCPU() g.Config.Linux.Resources.CPU.RealtimeRuntime = &time } // SetLinuxResourcesCPURealtimePeriod sets g.Config.Linux.Resources.CPU.RealtimePeriod. func (g *Generator) SetLinuxResourcesCPURealtimePeriod(period uint64) { - g.initConfigLinuxResourcesCPU() + g.InitConfigLinuxResourcesCPU() g.Config.Linux.Resources.CPU.RealtimePeriod = &period } // SetLinuxResourcesCPUCpus sets g.Config.Linux.Resources.CPU.Cpus. func (g *Generator) SetLinuxResourcesCPUCpus(cpus string) { - g.initConfigLinuxResourcesCPU() + g.InitConfigLinuxResourcesCPU() g.Config.Linux.Resources.CPU.Cpus = cpus } // SetLinuxResourcesCPUMems sets g.Config.Linux.Resources.CPU.Mems. func (g *Generator) SetLinuxResourcesCPUMems(mems string) { - g.initConfigLinuxResourcesCPU() + g.InitConfigLinuxResourcesCPU() g.Config.Linux.Resources.CPU.Mems = mems } diff --git a/validation/linux_cgroups_cpus.go b/validation/linux_cgroups_cpus.go index 2bc050913..a13707580 100644 --- a/validation/linux_cgroups_cpus.go +++ b/validation/linux_cgroups_cpus.go @@ -2,68 +2,132 @@ package main import ( "fmt" + "os" + "path/filepath" + "runtime" "github.com/mndrix/tap-go" - rspec "github.com/opencontainers/runtime-spec/specs-go" "github.com/opencontainers/runtime-tools/cgroups" "github.com/opencontainers/runtime-tools/validation/util" ) -func main() { - var shares uint64 = 1024 - var period uint64 = 100000 - var quota int64 = 50000 - var cpus, mems string = "0-1", "0" - g, err := util.GetDefaultGenerator() - if err != nil { - util.Fatal(err) +const ( + defaultRealtimePeriod uint64 = 1000000 + defaultRealtimeRuntime int64 = 950000 +) + +func testCPUCgroups() error { + t := tap.New() + t.Header(0) + defer t.AutoPlan() + + CPUrange := fmt.Sprintf("0-%d", runtime.NumCPU()-1) + + // Test with different combinations of values. + // NOTE: most systems have only one memory node (mems=="0"), so we cannot + // simply test with multiple values of mems. + cases := []struct { + shares uint64 + period uint64 + quota int64 + cpus string + mems string + }{ + {1024, 100000, 50000, "0", "0"}, + {1024, 100000, 50000, CPUrange, "0"}, + {1024, 100000, 200000, "0", "0"}, + {1024, 100000, 200000, CPUrange, "0"}, + {1024, 500000, 50000, "0", "0"}, + {1024, 500000, 50000, CPUrange, "0"}, + {1024, 500000, 200000, "0", "0"}, + {1024, 500000, 200000, CPUrange, "0"}, + {2048, 100000, 50000, "0", "0"}, + {2048, 100000, 50000, CPUrange, "0"}, + {2048, 100000, 200000, "0", "0"}, + {2048, 100000, 200000, CPUrange, "0"}, + {2048, 500000, 50000, "0", "0"}, + {2048, 500000, 50000, CPUrange, "0"}, + {2048, 500000, 200000, "0", "0"}, + {2048, 500000, 200000, CPUrange, "0"}, } - g.SetLinuxCgroupsPath(cgroups.AbsCgroupPath) - g.SetLinuxResourcesCPUShares(shares) - g.SetLinuxResourcesCPUQuota(quota) - g.SetLinuxResourcesCPUPeriod(period) - g.SetLinuxResourcesCPUCpus(cpus) - g.SetLinuxResourcesCPUMems(mems) - err = util.RuntimeOutsideValidate(g, nil, func(config *rspec.Spec, t *tap.T, state *rspec.State) error { - cg, err := cgroups.FindCgroup() - if err != nil { - return err - } - lcd, err := cg.GetCPUData(state.Pid, config.Linux.CgroupsPath) + + for _, c := range cases { + g, err := util.GetDefaultGenerator() if err != nil { - return err + return fmt.Errorf("cannot get default config from generator: %v", err) } - if lcd.Shares == nil { - return fmt.Errorf("unable to get cpu shares, lcd.Shares == %v", lcd.Shares) + g.SetLinuxCgroupsPath(cgroups.AbsCgroupPath) + + if c.shares > 0 { + g.SetLinuxResourcesCPUShares(c.shares) } - if *lcd.Shares != shares { - return fmt.Errorf("cpus shares limit is not set correctly, expect: %d, actual: %d", shares, *lcd.Shares) + + if c.period > 0 { + g.SetLinuxResourcesCPUPeriod(c.period) } - if lcd.Quota == nil { - return fmt.Errorf("unable to get cpu quota, lcd.Quota == %v", lcd.Quota) + if c.quota > 0 { + g.SetLinuxResourcesCPUQuota(c.quota) } - if *lcd.Quota != quota { - return fmt.Errorf("cpus quota is not set correctly, expect: %d, actual: %d", quota, *lcd.Quota) + + if c.cpus != "" { + g.SetLinuxResourcesCPUCpus(c.cpus) } - if lcd.Period == nil { - return fmt.Errorf("unable to get cpu period, lcd.Period == %v", lcd.Period) + if c.mems != "" { + g.SetLinuxResourcesCPUMems(c.mems) } - if *lcd.Period != period { - return fmt.Errorf("cpus period is not set correctly, expect: %d, actual: %d", period, *lcd.Period) + + // NOTE: On most systems where CONFIG_RT_GROUP & CONFIG_RT_GROUP_SCHED are not enabled, + // the following tests will fail, because sysfs knobs like + // /sys/fs/cgroup/cpu,cpuacct/cpu.rt_{period,runtime}_us do not exist. + // So we need to check if the sysfs knobs exist before setting the variables. + if _, err := os.Stat(filepath.Join(util.CPUCgroupPrefix, "cpu.rt_period_us")); !os.IsNotExist(err) { + g.SetLinuxResourcesCPURealtimePeriod(defaultRealtimePeriod) } - if lcd.Cpus != cpus { - return fmt.Errorf("cpus cpus is not set correctly, expect: %s, actual: %s", cpus, lcd.Cpus) + + if _, err := os.Stat(filepath.Join(util.CPUCgroupPrefix, "cpu.rt_runtime_us")); !os.IsNotExist(err) { + g.SetLinuxResourcesCPURealtimeRuntime(defaultRealtimeRuntime) } - if lcd.Mems != mems { - return fmt.Errorf("cpus mems is not set correctly, expect: %s, actual: %s", mems, lcd.Mems) + + if err := util.RuntimeOutsideValidate(g, t, util.ValidateLinuxResourcesCPU); err != nil { + return fmt.Errorf("cannot validate CPU cgroups: %v", err) } - return nil - }) + } + + return nil +} +func testEmptyCPU() error { + t := tap.New() + t.Header(0) + defer t.AutoPlan() + + g, err := util.GetDefaultGenerator() if err != nil { + return fmt.Errorf("cannot get default config from generator: %v", err) + } + g.InitConfigLinuxResourcesCPU() + g.SetLinuxCgroupsPath(cgroups.AbsCgroupPath) + + if err := util.RuntimeOutsideValidate(g, t, util.ValidateLinuxResourcesCPUEmpty); err != nil { + return fmt.Errorf("cannot validate empty CPU cgroups: %v", err) + } + + return nil +} + +func main() { + if "linux" != runtime.GOOS { + util.Fatal(fmt.Errorf("linux-specific cgroup test")) + } + + if err := testCPUCgroups(); err != nil { + util.Fatal(err) + } + + if err := testEmptyCPU(); err != nil { util.Fatal(err) } } diff --git a/validation/util/linux_resources_cpus.go b/validation/util/linux_resources_cpus.go new file mode 100644 index 000000000..8adb8a0e2 --- /dev/null +++ b/validation/util/linux_resources_cpus.go @@ -0,0 +1,137 @@ +package util + +import ( + "fmt" + "io/ioutil" + "path/filepath" + "runtime" + "strconv" + "strings" + + "github.com/mndrix/tap-go" + rspec "github.com/opencontainers/runtime-spec/specs-go" + "github.com/opencontainers/runtime-tools/cgroups" +) + +const ( + // CPUCgroupPrefix is default path prefix where CPU cgroups are created + CPUCgroupPrefix string = "/sys/fs/cgroup/cpu,cpuacct" +) + +// ValidateLinuxResourcesCPU validates if Linux.Resources.CPU is set to +// correct values, the same as given values in the config. +func ValidateLinuxResourcesCPU(config *rspec.Spec, t *tap.T, state *rspec.State) error { + cg, err := cgroups.FindCgroup() + t.Ok((err == nil), "find cpu cgroup") + if err != nil { + t.Diagnostic(err.Error()) + return nil + } + + lcd, err := cg.GetCPUData(state.Pid, config.Linux.CgroupsPath) + t.Ok((err == nil), "get cpu cgroup data") + if err != nil { + t.Diagnostic(err.Error()) + return nil + } + + if lcd.Shares == nil || config.Linux.Resources.CPU.Shares == nil { + t.Diagnostic(fmt.Sprintf("unable to get cpu shares, lcd.Shares == %v, config.Linux.Resources.CPU.Shares == %v", lcd.Shares, config.Linux.Resources.CPU.Shares)) + return nil + } + t.Ok(*lcd.Shares == *config.Linux.Resources.CPU.Shares, "cpu shares is set correctly") + t.Diagnosticf("expect: %d, actual: %d", *config.Linux.Resources.CPU.Shares, *lcd.Shares) + + if lcd.Period == nil || config.Linux.Resources.CPU.Period == nil { + t.Diagnostic(fmt.Sprintf("unable to get cpu period, lcd.Period == %v, config.Linux.Resources.CPU.Period == %v", lcd.Period, config.Linux.Resources.CPU.Period)) + return nil + } + t.Ok(*lcd.Period == *config.Linux.Resources.CPU.Period, "cpu period is set correctly") + t.Diagnosticf("expect: %d, actual: %d", *config.Linux.Resources.CPU.Period, *lcd.Period) + + if lcd.Quota == nil || config.Linux.Resources.CPU.Quota == nil { + t.Diagnostic(fmt.Sprintf("unable to get cpu quota, lcd.Quota == %v, config.Linux.Resources.CPU.Quota == %v", lcd.Quota, config.Linux.Resources.CPU.Quota)) + return nil + } + t.Ok(*lcd.Quota == *config.Linux.Resources.CPU.Quota, "cpu quota is set correctly") + t.Diagnosticf("expect: %d, actual: %d", *config.Linux.Resources.CPU.Quota, *lcd.Quota) + + t.Ok(lcd.Cpus == config.Linux.Resources.CPU.Cpus, "cpu cpus is set correctly") + t.Diagnosticf("expect: %s, actual: %s", config.Linux.Resources.CPU.Cpus, lcd.Cpus) + + t.Ok(lcd.Mems == config.Linux.Resources.CPU.Mems, "cpu mems is set correctly") + t.Diagnosticf("expect: %s, actual: %s", config.Linux.Resources.CPU.Mems, lcd.Mems) + + return nil +} + +// ValidateLinuxResourcesCPUEmpty validates Linux.Resources.CPU is set to +// correct values, when each value are set to the default ones. +func ValidateLinuxResourcesCPUEmpty(config *rspec.Spec, t *tap.T, state *rspec.State) error { + outShares, err := ioutil.ReadFile(filepath.Join(CPUCgroupPrefix, "cpu.shares")) + if err != nil { + return nil + } + sh, _ := strconv.Atoi(strings.TrimSpace(string(outShares))) + defaultShares := uint64(sh) + + outPeriod, err := ioutil.ReadFile(filepath.Join(CPUCgroupPrefix, "cpu.cfs_period_us")) + if err != nil { + return nil + } + pe, _ := strconv.Atoi(strings.TrimSpace(string(outPeriod))) + defaultPeriod := uint64(pe) + + outQuota, err := ioutil.ReadFile(filepath.Join(CPUCgroupPrefix, "cpu.cfs_quota_us")) + if err != nil { + return nil + } + qu, _ := strconv.Atoi(strings.TrimSpace(string(outQuota))) + defaultQuota := int64(qu) + + defaultCpus := fmt.Sprintf("0-%d", runtime.NumCPU()-1) + defaultMems := "0" + + cg, err := cgroups.FindCgroup() + t.Ok((err == nil), "find cpu cgroup") + if err != nil { + t.Diagnostic(err.Error()) + return nil + } + + lcd, err := cg.GetCPUData(state.Pid, config.Linux.CgroupsPath) + t.Ok((err == nil), "get cpu cgroup data") + if err != nil { + t.Diagnostic(err.Error()) + return nil + } + + if lcd.Shares == nil { + t.Diagnostic(fmt.Sprintf("unable to get cpu shares, lcd.Shares == %v", lcd.Shares)) + return nil + } + t.Ok(*lcd.Shares == defaultShares, "cpu shares is set correctly") + t.Diagnosticf("expect: %d, actual: %d", defaultShares, *lcd.Shares) + + if lcd.Period == nil { + t.Diagnostic(fmt.Sprintf("unable to get cpu period, lcd.Period == %v", lcd.Period)) + return nil + } + t.Ok(*lcd.Period == defaultPeriod, "cpu period is set correctly") + t.Diagnosticf("expect: %d, actual: %d", defaultPeriod, *lcd.Period) + + if lcd.Quota == nil { + t.Diagnostic(fmt.Sprintf("unable to get cpu quota, lcd.Quota == %v", lcd.Quota)) + return nil + } + t.Ok(*lcd.Quota == defaultQuota, "cpu quota is set correctly") + t.Diagnosticf("expect: %d, actual: %d", defaultQuota, *lcd.Quota) + + t.Ok(lcd.Cpus == defaultCpus, "cpu cpus is set correctly") + t.Diagnosticf("expect: %s, actual: %s", defaultCpus, lcd.Cpus) + + t.Ok(lcd.Mems == defaultMems, "cpu mems is set correctly") + t.Diagnosticf("expect: %s, actual: %s", defaultMems, lcd.Mems) + + return nil +}