From 404ac9677b1656321b27133b7c4a8671e3a50f7a Mon Sep 17 00:00:00 2001 From: Nashwan Azhari Date: Fri, 7 Jan 2022 23:56:41 +0200 Subject: [PATCH] Parametrize benchmark settings. Signed-off-by: Nashwan Azhari --- cmd/critest/cri_test.go | 1 - pkg/benchmark/benchmark.go | 9 ++++++ pkg/benchmark/container.go | 46 ++++++++++++++++----------- pkg/benchmark/pod.go | 35 +++++++++++++------- pkg/framework/test_context.go | 60 +++++++++++++++++++++++++---------- pkg/framework/util.go | 21 +++++++++++- 6 files changed, 124 insertions(+), 48 deletions(-) diff --git a/cmd/critest/cri_test.go b/cmd/critest/cri_test.go index 7956618886..cf4aee19e7 100644 --- a/cmd/critest/cri_test.go +++ b/cmd/critest/cri_test.go @@ -174,7 +174,6 @@ func TestCRISuite(t *testing.T) { flag.Set("ginkgo.focus", "benchmark") flag.Set("ginkgo.succinct", "true") } else { - // Skip benchmark measurements for validation tests. flag.Set("ginkgo.skipMeasurements", "true") } if *parallel > 1 { diff --git a/pkg/benchmark/benchmark.go b/pkg/benchmark/benchmark.go index bfed633e5c..5e5b2ce1b6 100644 --- a/pkg/benchmark/benchmark.go +++ b/pkg/benchmark/benchmark.go @@ -33,6 +33,15 @@ import ( . "github.com/onsi/gomega" ) +// Transforms a slice of `time.Duration`s into their `int64` nanosecond representations. +func getNanosecondsForDurations(durations []time.Duration) []int64 { + var ns []int64 + for _, duration := range durations { + ns = append(ns, duration.Nanoseconds()) + } + return ns +} + // TestPerformance checks configuration parameters (specified through flags) and then runs // benchmark tests using the Ginkgo runner. // If a "report directory" is specified, one or more JUnit test reports will be diff --git a/pkg/benchmark/container.go b/pkg/benchmark/container.go index a6d3a11b64..f09944cc76 100644 --- a/pkg/benchmark/container.go +++ b/pkg/benchmark/container.go @@ -1,6 +1,3 @@ -//go:build container -// +build container - /* Copyright 2021 The Kubernetes Authors. @@ -20,19 +17,20 @@ limitations under the License. package benchmark import ( + "encoding/json" + "io/ioutil" + "path" + + "github.com/golang/glog" "github.com/kubernetes-sigs/cri-tools/pkg/framework" + . "github.com/onsi/ginkgo" "github.com/onsi/gomega/gmeasure" internalapi "k8s.io/cri-api/pkg/apis" - runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1" - - "encoding/json" - "fmt" - . "github.com/onsi/ginkgo" - "io/ioutil" + runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" ) type ContainerExperimentData struct { - CreateContainer, StatusContainer, StopContainer, RemoveContainer, StartContainer string + CreateContainer, StatusContainer, StopContainer, RemoveContainer, StartContainer []int64 } var _ = framework.KubeDescribe("Container", func() { @@ -92,18 +90,30 @@ var _ = framework.KubeDescribe("Container", func() { By("delete PodSandbox") rc.RemovePodSandbox(podID) - }, gmeasure.SamplingConfig{N: 200, NumParallel: 1}) + }, gmeasure.SamplingConfig{N: framework.TestContext.BenchmarkingParams.ContainersNumber, NumParallel: framework.TestContext.BenchmarkingParams.ContainersNumberParallel}) data := ContainerExperimentData{ - CreateContainer: fmt.Sprintf("%v", experiment.Get("CreateContainer").Durations), - StatusContainer: fmt.Sprintf("%v", experiment.Get("StatusContainer").Durations), - StopContainer: fmt.Sprintf("%v", experiment.Get("StopContainer").Durations), - RemoveContainer: fmt.Sprintf("%v", experiment.Get("RemoveContainer").Durations), - StartContainer: fmt.Sprintf("%v", experiment.Get("StartContainer").Durations), + CreateContainer: getNanosecondsForDurations(experiment.Get("CreateContainer").Durations), + StartContainer: getNanosecondsForDurations(experiment.Get("StartContainer").Durations), + StatusContainer: getNanosecondsForDurations(experiment.Get("StatusContainer").Durations), + StopContainer: getNanosecondsForDurations(experiment.Get("StopContainer").Durations), + RemoveContainer: getNanosecondsForDurations(experiment.Get("RemoveContainer").Durations), } - file, _ := json.MarshalIndent(data, "", " ") - _ = ioutil.WriteFile("c:/experiment_container.json", file, 0644) + if framework.TestContext.BenchmarkingOutputDir != "" { + filepath := path.Join(framework.TestContext.BenchmarkingOutputDir, "container_benchmark_data.json") + data, err := json.MarshalIndent(data, "", " ") + if err == nil { + err = ioutil.WriteFile(filepath, data, 0644) + if err != nil { + glog.Errorf("Failed to write container benchmark data: %v", filepath) + } + } else { + glog.Errorf("Failed to serialize benchmark data: %v", err) + } + } else { + glog.Infof("No benchmarking output dir provided, skipping writing benchmarking resulsts.") + } }) }) diff --git a/pkg/benchmark/pod.go b/pkg/benchmark/pod.go index f4ffdc143b..5c1461c111 100644 --- a/pkg/benchmark/pod.go +++ b/pkg/benchmark/pod.go @@ -1,6 +1,3 @@ -//go:build pod -// +build pod - /* Copyright 2021 The Kubernetes Authors. @@ -20,6 +17,9 @@ limitations under the License. package benchmark import ( + "path" + + "github.com/golang/glog" "github.com/kubernetes-sigs/cri-tools/pkg/framework" . "github.com/onsi/ginkgo" @@ -28,7 +28,6 @@ import ( runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1" "encoding/json" - "fmt" "io/ioutil" ) @@ -37,7 +36,7 @@ const ( ) type ExperimentData struct { - CreatePod, StatusPod, StopPod, RemovePod string + CreatePod, StatusPod, StopPod, RemovePod []int64 } var _ = framework.KubeDescribe("PodSandbox", func() { @@ -92,17 +91,29 @@ var _ = framework.KubeDescribe("PodSandbox", func() { stopwatch.Record("RemovePod") framework.ExpectNoError(err, "failed to remove PodSandbox: %v", err) - }, gmeasure.SamplingConfig{N: 1000, NumParallel: 1}) + }, gmeasure.SamplingConfig{N: framework.TestContext.BenchmarkingParams.PodsNumber, NumParallel: framework.TestContext.BenchmarkingParams.PodsNumberParallel}) data := ExperimentData{ - CreatePod: fmt.Sprintf("%v", experiment.Get("CreatePod").Durations), - StatusPod: fmt.Sprintf("%v", experiment.Get("StatusPod").Durations), - StopPod: fmt.Sprintf("%v", experiment.Get("StopPod").Durations), - RemovePod: fmt.Sprintf("%v", experiment.Get("RemovePod").Durations), + CreatePod: getNanosecondsForDurations(experiment.Get("CreatePod").Durations), + StatusPod: getNanosecondsForDurations(experiment.Get("StatusPod").Durations), + StopPod: getNanosecondsForDurations(experiment.Get("StopPod").Durations), + RemovePod: getNanosecondsForDurations(experiment.Get("RemovePod").Durations), } - file, _ := json.MarshalIndent(data, "", " ") - _ = ioutil.WriteFile("c:/experiment_old_hcsshim.json", file, 0644) + if framework.TestContext.BenchmarkingOutputDir != "" { + filepath := path.Join(framework.TestContext.BenchmarkingOutputDir, "pod_benchmark_data.json") + data, err := json.MarshalIndent(data, "", " ") + if err == nil { + err = ioutil.WriteFile(filepath, data, 0644) + if err != nil { + glog.Errorf("Failed to write container benchmark data: %v", filepath) + } + } else { + glog.Errorf("Failed to serialize benchmark data: %v", err) + } + } else { + glog.Infof("No benchmarking out dir provided, skipping writing benchmarking resulsts.") + } }) }) diff --git a/pkg/framework/test_context.go b/pkg/framework/test_context.go index 1e8d4bbc0f..ce04e9a304 100644 --- a/pkg/framework/test_context.go +++ b/pkg/framework/test_context.go @@ -25,7 +25,11 @@ import ( "time" "github.com/onsi/ginkgo/config" - "gopkg.in/yaml.v3" +) + +var ( + testImagesFilePath string + benchamrkSettingFilePath string ) // TestImageList aggregates references to the images used in tests. @@ -34,6 +38,24 @@ type TestImageList struct { WebServerTestImage string `yaml:"webServerTestImage"` } +// BenchmarkingParamsType is the type of benchmarking-related params. +type BenchmarkingParamsType struct { + // ContainersNumber is the number of Containers to run as part of + // the container-related benchmarks. + ContainersNumber int `yaml:"containersNumber"` + + // ContainersNumberParallel is the maximum number of container-related benchmarks + // to run in parallel. + ContainersNumberParallel int `yaml:"containersNumberParallel"` + + // PodsNumber is the number of Pods to run as part of the pod-related benchmarks. + PodsNumber int `yaml:"podsNumber"` + + // PodsNumberParallel is the maximum number of pod -related benchmarks + // to run in parallel. + PodsNumberParallel int `yaml:"podsNumberParallel"` +} + // TestContextType is the type of test context. type TestContextType struct { // Report related settings. @@ -49,11 +71,11 @@ type TestContextType struct { RuntimeHandler string // Test images-related settings. - TestImagesFilePath string - TestImageList TestImageList + TestImageList TestImageList - // Benchmark setting. - Number int + // Benchmarking settings. + BenchmarkingOutputDir string + BenchmarkingParams BenchmarkingParamsType // Test configuration. IsLcow bool @@ -81,7 +103,7 @@ func RegisterFlags() { flag.StringVar(&TestContext.ReportPrefix, "report-prefix", "", "Optional prefix for JUnit XML reports. Default is empty, which doesn't prepend anything to the default name.") flag.StringVar(&TestContext.ReportDir, "report-dir", "", "Path to the directory where the JUnit XML reports should be saved. Default is empty, which doesn't generate these reports.") flag.StringVar(&TestContext.ImageServiceAddr, "image-endpoint", "", "Image service socket for client to connect.") - flag.StringVar(&TestContext.TestImagesFilePath, "test-images-file", "", "Optional path to a YAML file containing references to custom container images to be used in tests.") + flag.StringVar(&testImagesFilePath, "test-images-file", "", "Optional path to a YAML file containing references to custom container images to be used in tests.") flag.DurationVar(&TestContext.ImageServiceTimeout, "image-service-timeout", 300*time.Second, "Timeout when trying to connect to image service.") svcaddr := "unix:///var/run/dockershim.sock" @@ -94,7 +116,9 @@ func RegisterFlags() { flag.StringVar(&TestContext.RuntimeServiceAddr, "runtime-endpoint", svcaddr, "Runtime service socket for client to connect.") flag.DurationVar(&TestContext.RuntimeServiceTimeout, "runtime-service-timeout", 300*time.Second, "Timeout when trying to connect to a runtime service.") flag.StringVar(&TestContext.RuntimeHandler, "runtime-handler", "", "Runtime handler to use in the test.") - flag.IntVar(&TestContext.Number, "number", 5, "Number of PodSandbox/container in listing benchmark test.") + + flag.StringVar(&benchamrkSettingFilePath, "benchmarking-params-file", "", "Optional path to a YAML file specifying benchmarking configuration options.") + flag.StringVar(&TestContext.BenchmarkingOutputDir, "benchmarking-output-dir", "", "Optional path to a directory in which benchmarking data should be placed.") if runtime.GOOS == "windows" { flag.BoolVar(&TestContext.IsLcow, "lcow", false, "Run Linux container on Windows tests instead of Windows container tests") @@ -104,21 +128,25 @@ func RegisterFlags() { flag.StringVar(&TestContext.RegistryPrefix, "registry-prefix", DefaultRegistryPrefix, "A possible registry prefix added to all images, like 'localhost:5000/'") } -// Loads the custom images mapping file (if defined) into the TestContextType. -func (tc TestContextType) LoadCustomImagesFileIntoTestingContext() error { - Logf("Testing context container image list file: %s", TestContext.TestImagesFilePath) - if TestContext.TestImagesFilePath != "" { - fileContent, err := os.ReadFile(TestContext.TestImagesFilePath) +// Loads any external file-based parameters into the TestContextType. +func (tc TestContextType) LoadYamlConfigFiles() error { + // Attempt to load cusom images file: + if testImagesFilePath != "" { + err := LoadYamlFile(testImagesFilePath, &TestContext.TestImageList) if err != nil { - return fmt.Errorf("error reading '%v' file contents: %v", TestContext.TestImagesFilePath, err) + return fmt.Errorf("Error loading custom test images file: %v", err) } + } + Logf("Testing context container image list: %+v", TestContext.TestImageList) - err = yaml.Unmarshal(fileContent, &TestContext.TestImageList) + // Attempt to load benchmark settings file: + if benchamrkSettingFilePath != "" { + err := LoadYamlFile(benchamrkSettingFilePath, &TestContext.BenchmarkingParams) if err != nil { - return fmt.Errorf("error unmarshalling '%v' YAML file: %v", TestContext.TestImagesFilePath, err) + return err } } + Logf("Testing context benchmarking params: %+v", TestContext.BenchmarkingParams) - Logf("Testing context container image list: %+v", TestContext.TestImageList) return nil } diff --git a/pkg/framework/util.go b/pkg/framework/util.go index 6859657977..87275143a6 100644 --- a/pkg/framework/util.go +++ b/pkg/framework/util.go @@ -18,6 +18,7 @@ package framework import ( "fmt" + "os" "runtime" "strings" "sync" @@ -25,6 +26,7 @@ import ( "github.com/docker/distribution/reference" "github.com/pborman/uuid" + "gopkg.in/yaml.v3" internalapi "k8s.io/cri-api/pkg/apis" runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1" "k8s.io/kubernetes/pkg/kubelet/cri/remote" @@ -115,7 +117,7 @@ var _ = BeforeSuite(func() { } // Load any custom image definitions: - err := TestContext.LoadCustomImagesFileIntoTestingContext() + err := TestContext.LoadYamlConfigFiles() if err != nil { panic(err) } @@ -355,3 +357,20 @@ func PullPublicImage(c internalapi.ImageManagerService, imageName string, podCon ExpectNoError(err, "failed to pull image: %v", err) return id } + +// LoadYamlFile attempts to load the given YAML file into the given struct. +func LoadYamlFile(filepath string, obj interface{}) error { + Logf("Attempting to load YAML file %q into %+v", filepath, obj) + fileContent, err := os.ReadFile(filepath) + if err != nil { + return fmt.Errorf("error reading %q file contents: %v", filepath, err) + } + + err = yaml.Unmarshal(fileContent, obj) + if err != nil { + return fmt.Errorf("error unmarshalling %q YAML file: %v", filepath, err) + } + + Logf("Successfully loaded YAML file %q into %+v", filepath, obj) + return nil +}