Skip to content

Commit

Permalink
PTX-18776 Adding validating for stork-scheduler image (#1145)
Browse files Browse the repository at this point in the history 
Signed-off-by: nikolaypopov <nikolay.popov86@gmail.com>
  • Loading branch information
@nikolaypopov
nikolaypopov committed Jul 23, 2023
1 parent 2056026 commit 9b862bc
Show file tree
Hide file tree
Showing 3 changed files with 98 additions and 58 deletions.
136 changes: 88 additions & 48 deletions pkg/util/test/util.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1370,7 +1370,7 @@ func validateComponents(pxImageList map[string]string, cluster *corev1.StorageCl
}

// Validate Stork components and images
if err := ValidateStork(pxImageList, cluster, k8sVersion, timeout, interval); err != nil {
if err := ValidateStork(pxImageList, cluster, timeout, interval); err != nil {
return err
}

Expand Down Expand Up @@ -1595,7 +1595,7 @@ func ValidatePvcControllerDisabled(pvcControllerDp *appsv1.Deployment, timeout,
}

// ValidateStork validates Stork components and images
func ValidateStork(pxImageList map[string]string, cluster *corev1.StorageCluster, k8sVersion string, timeout, interval time.Duration) error {
func ValidateStork(pxImageList map[string]string, cluster *corev1.StorageCluster, timeout, interval time.Duration) error {
logrus.Info("Validate Stork components")

storkDp := &appsv1.Deployment{
Expand All @@ -1614,14 +1614,14 @@ func ValidateStork(pxImageList map[string]string, cluster *corev1.StorageCluster

if cluster.Spec.Stork != nil && cluster.Spec.Stork.Enabled {
logrus.Debug("Stork is enabled in StorageCluster")
return ValidateStorkEnabled(pxImageList, cluster, storkDp, storkSchedulerDp, k8sVersion, timeout, interval)
return ValidateStorkEnabled(pxImageList, cluster, storkDp, storkSchedulerDp, timeout, interval)
}
logrus.Debug("Stork is disabled in StorageCluster")
return ValidateStorkDisabled(cluster, storkDp, storkSchedulerDp, timeout, interval)
}

// ValidateStorkEnabled validates that all Stork components are enabled/created
func ValidateStorkEnabled(pxImageList map[string]string, cluster *corev1.StorageCluster, storkDp, storkSchedulerDp *appsv1.Deployment, k8sVersion string, timeout, interval time.Duration) error {
func ValidateStorkEnabled(pxImageList map[string]string, cluster *corev1.StorageCluster, storkDp, storkSchedulerDp *appsv1.Deployment, timeout, interval time.Duration) error {
logrus.Info("Validate Stork components are enabled")

t := func() (interface{}, bool, error) {
Expand Down Expand Up @@ -1654,43 +1654,6 @@ func ValidateStorkEnabled(pxImageList map[string]string, cluster *corev1.Storage
return nil, true, err
}

// Validate stork-scheduler deployment and pods
if err := validateDeployment(storkSchedulerDp, timeout, interval); err != nil {
return nil, true, err
}

K8sVer1_22, _ := version.NewVersion("1.22")
k8sMinVersionForKubeSchedulerConfiguration, _ := version.NewVersion("1.23")
kubeVersion, _, err := GetFullVersion()
if err != nil {
return nil, true, err
}

opVersion, _ := GetPxOperatorVersion()
if opVersion.LessThan(minOpVersionForKubeSchedConfig) {
if kubeVersion != nil && kubeVersion.GreaterThanOrEqual(K8sVer1_22) {
// Image tag for stork-scheduler is hardcoded to v1.21.4 for clusters 1.22 and up for Operator version 1.10.1 and below
if err = validateImageTag("v1.21.4", cluster.Namespace, map[string]string{"name": "stork-scheduler"}); err != nil {
return nil, true, err
}
} else {
if err = validateImageTag(k8sVersion, cluster.Namespace, map[string]string{"name": "stork-scheduler"}); err != nil {
return nil, true, err
}
}
} else {
if kubeVersion != nil && kubeVersion.GreaterThanOrEqual(K8sVer1_22) && kubeVersion.LessThan(k8sMinVersionForKubeSchedulerConfiguration) {
// Image tag for stork-scheduler is hardcoded to v1.21.4 for clusters 1.22
if err = validateImageTag("v1.21.4", cluster.Namespace, map[string]string{"name": "stork-scheduler"}); err != nil {
return nil, true, err
}
} else {
if err = validateImageTag(k8sVersion, cluster.Namespace, map[string]string{"name": "stork-scheduler"}); err != nil {
return nil, true, err
}
}
}

// Validate webhook-controller arguments
if err := validateStorkWebhookController(cluster.Spec.Stork.Args, storkDp, timeout, interval); err != nil {
return nil, true, err
Expand All @@ -1706,8 +1669,8 @@ func ValidateStorkEnabled(pxImageList map[string]string, cluster *corev1.Storage
return nil, true, err
}

// Validate stork scheduler deployment pod topology spread constraints
if err := validatePodTopologySpreadConstraints(storkSchedulerDp, timeout, interval); err != nil {
// Validate stork-scheduler deployment and image
if err := ValidateStorkScheduler(pxImageList, cluster, storkSchedulerDp, timeout, interval); err != nil {
return nil, true, err
}

Expand All @@ -1721,6 +1684,83 @@ func ValidateStorkEnabled(pxImageList map[string]string, cluster *corev1.Storage
return nil
}

// ValidateStorkScheduler validates stork-scheduler deployment and container images inside pods
func ValidateStorkScheduler(pxImageList map[string]string, cluster *corev1.StorageCluster, storkSchedulerDp *appsv1.Deployment, timeout, interval time.Duration) error {
logrus.Info("Validate stork-scheduler deployment and image")

// Validate stork-scheduler deployment and pods
if err := validateDeployment(storkSchedulerDp, timeout, interval); err != nil {
return err
}

// Get stork-scheduler pods
pods, err := coreops.Instance().GetPods(cluster.Namespace, map[string]string{"name": "stork-scheduler"})
if err != nil {
return err
}

K8sVer1_22, _ := version.NewVersion("1.22")
k8sMinVersionForKubeSchedulerConfiguration, _ := version.NewVersion("1.23")
kubeVersion, _, err := GetFullVersion()
if err != nil {
return err
}

// Figure out what default registry to use for stork-scheduler image, based on PX Operator version
storkSchedulerImageName := "k8s.gcr.io/kube-scheduler-amd64"
opVersion, _ := GetPxOperatorVersion()
if opVersion.GreaterThanOrEqual(opVer23_3) {
storkSchedulerImageName = "registry.k8s.io/kube-scheduler-amd64"
}

// Check if stork-scheduler image was explicitly set in the px-version configmap
explicitStorkSchedulerImage := ""
if value, ok := pxImageList["kubeScheduler"]; ok {
explicitStorkSchedulerImage = value
}

if len(explicitStorkSchedulerImage) > 0 {
logrus.Debugf("Image for stork-scheduler was explicitly set in the px-version configmap to [%s]", explicitStorkSchedulerImage)
// Use this image as is, since it was explicitly set
if err := validateContainerImageInsidePods(cluster, explicitStorkSchedulerImage, "stork-scheduler", pods); err != nil {
return err
}
} else {
if opVersion.LessThan(minOpVersionForKubeSchedConfig) {
if kubeVersion != nil && kubeVersion.GreaterThanOrEqual(K8sVer1_22) {
// This image should have v1.21.4 tag
if err := validateContainerImageInsidePods(cluster, fmt.Sprintf("%s:v1.21.4", storkSchedulerImageName), "stork-scheduler", pods); err != nil {
return err
}
} else {
// This image should have k8s version tag
if err := validateContainerImageInsidePods(cluster, fmt.Sprintf("%s:v%s", storkSchedulerImageName, kubeVersion.String()), "stork-scheduler", pods); err != nil {
return err
}
}
} else {
if kubeVersion != nil && kubeVersion.GreaterThanOrEqual(K8sVer1_22) && kubeVersion.LessThan(k8sMinVersionForKubeSchedulerConfiguration) {
// This image should have v1.21.4 tag
if err := validateContainerImageInsidePods(cluster, fmt.Sprintf("%s:v1.21.4", storkSchedulerImageName), "stork-scheduler", pods); err != nil {
return err
}
} else {
// This image should have k8s version tag
if err := validateContainerImageInsidePods(cluster, fmt.Sprintf("%s:v%s", storkSchedulerImageName, kubeVersion.String()), "stork-scheduler", pods); err != nil {
return err
}
}
}
}

// Validate stork-scheduler deployment pod topology spread constraints
if err := validatePodTopologySpreadConstraints(storkSchedulerDp, timeout, interval); err != nil {
return err
}

return nil
}

// ValidateStorkDisabled validates that all Stork components are disabled/deleted
func ValidateStorkDisabled(cluster *corev1.StorageCluster, storkDp, storkSchedulerDp *appsv1.Deployment, timeout, interval time.Duration) error {
logrus.Info("Validate Stork components are disabled")
Expand Down Expand Up @@ -2535,7 +2575,7 @@ func validateCsiExtImages(cluster *corev1.StorageCluster, pxImageList map[string
}

func validateContainerImageInsidePods(cluster *corev1.StorageCluster, expectedImage, containerName string, pods *v1.PodList) error {
logrus.Infof("Validating image for %s container inside pod(s)", containerName)
logrus.Infof("Validating image for [%s] container inside pod(s)", containerName)

// Get PX Operator version
opVersion, _ := GetPxOperatorVersion()
Expand All @@ -2562,22 +2602,22 @@ func validateContainerImageInsidePods(cluster *corev1.StorageCluster, expectedIm
if container.Name == containerName {
foundImage = container.Image
if opVersion.GreaterThanOrEqual(opVer1_9_1) && foundImage == expectedImage {
logrus.Infof("Image inside %s[%s] matches, expected: %s, actual: %s", pod.Name, containerName, expectedImage, foundImage)
logrus.Infof("Image inside %s[%s] matches, expected: [%s], actual: [%s]", pod.Name, containerName, expectedImage, foundImage)
foundContainer = true
break
} else if strings.Contains(foundImage, expectedImage) {
logrus.Infof("Image inside %s[%s] matches, expected: %s, actual: %s", pod.Name, containerName, expectedImage, foundImage)
logrus.Infof("Image inside %s[%s] matches, expected: [%s], actual: [%s]", pod.Name, containerName, expectedImage, foundImage)
foundContainer = true
break
} else {
return fmt.Errorf("failed to match container %s[%s] image, expected: %s, actual: %s",
return fmt.Errorf("failed to match container %s[%s] image, expected: [%s], actual: [%s]",
pod.Name, containerName, expectedImage, foundImage)
}
}
}

if !foundContainer {
return fmt.Errorf("failed to match container %s[%s] image, expected: %s, actual: %s",
return fmt.Errorf("failed to match container %s[%s] image, expected: [%s], actual: [%s]",
pod.Name, containerName, expectedImage, foundImage)
}
}
Expand Down
16 changes: 8 additions & 8 deletions test/integration_test/basic_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -646,58 +646,58 @@ func BasicStorkRegression(tc *types.TestCase) func(*testing.T) {
cluster.Spec.Stork.Args["webhook-controller"] = "true"
return cluster
}
cluster = ci_utils.UpdateAndValidateStork(cluster, updateParamFunc, ci_utils.PxSpecImages, ci_utils.K8sVersion, t)
cluster = ci_utils.UpdateAndValidateStork(cluster, updateParamFunc, ci_utils.PxSpecImages, t)

logrus.Info("Disable Stork webhook-controller and validate StorageCluster")
updateParamFunc = func(cluster *corev1.StorageCluster) *corev1.StorageCluster {
cluster.Spec.Stork.Args["webhook-controller"] = "false"
return cluster
}
cluster = ci_utils.UpdateAndValidateStork(cluster, updateParamFunc, ci_utils.PxSpecImages, ci_utils.K8sVersion, t)
cluster = ci_utils.UpdateAndValidateStork(cluster, updateParamFunc, ci_utils.PxSpecImages, t)

logrus.Info("Remove Stork webhook-controller and validate StorageCluster")
updateParamFunc = func(cluster *corev1.StorageCluster) *corev1.StorageCluster {
delete(cluster.Spec.Stork.Args, "webhook-controller")
return cluster
}
cluster = ci_utils.UpdateAndValidateStork(cluster, updateParamFunc, ci_utils.PxSpecImages, ci_utils.K8sVersion, t)
cluster = ci_utils.UpdateAndValidateStork(cluster, updateParamFunc, ci_utils.PxSpecImages, t)

logrus.Info("Enable Stork hostNetwork and validate StorageCluster")
updateParamFunc = func(cluster *corev1.StorageCluster) *corev1.StorageCluster {
hostNetworkValue := true
cluster.Spec.Stork.HostNetwork = &hostNetworkValue
return cluster
}
cluster = ci_utils.UpdateAndValidateStork(cluster, updateParamFunc, ci_utils.PxSpecImages, ci_utils.K8sVersion, t)
cluster = ci_utils.UpdateAndValidateStork(cluster, updateParamFunc, ci_utils.PxSpecImages, t)

logrus.Info("Disable Stork hostNetwork and validate StorageCluster")
updateParamFunc = func(cluster *corev1.StorageCluster) *corev1.StorageCluster {
*cluster.Spec.Stork.HostNetwork = false
return cluster
}
cluster = ci_utils.UpdateAndValidateStork(cluster, updateParamFunc, ci_utils.PxSpecImages, ci_utils.K8sVersion, t)
cluster = ci_utils.UpdateAndValidateStork(cluster, updateParamFunc, ci_utils.PxSpecImages, t)

logrus.Info("Remove Stork hostNetwork and validate StorageCluster")
updateParamFunc = func(cluster *corev1.StorageCluster) *corev1.StorageCluster {
cluster.Spec.Stork.HostNetwork = nil
return cluster
}
cluster = ci_utils.UpdateAndValidateStork(cluster, updateParamFunc, ci_utils.PxSpecImages, ci_utils.K8sVersion, t)
cluster = ci_utils.UpdateAndValidateStork(cluster, updateParamFunc, ci_utils.PxSpecImages, t)

logrus.Info("Disable Stork and validate StorageCluster")
updateParamFunc = func(cluster *corev1.StorageCluster) *corev1.StorageCluster {
cluster.Spec.Stork.Enabled = false
return cluster
}
cluster = ci_utils.UpdateAndValidateStork(cluster, updateParamFunc, ci_utils.PxSpecImages, ci_utils.K8sVersion, t)
cluster = ci_utils.UpdateAndValidateStork(cluster, updateParamFunc, ci_utils.PxSpecImages, t)
require.False(t, cluster.Spec.Stork.Enabled, "failed to validate Stork is disabled: expected: false, actual: %v", cluster.Spec.Stork.Enabled)

logrus.Info("Enable Stork and validate StorageCluster")
updateParamFunc = func(cluster *corev1.StorageCluster) *corev1.StorageCluster {
cluster.Spec.Stork.Enabled = true
return cluster
}
cluster = ci_utils.UpdateAndValidateStork(cluster, updateParamFunc, ci_utils.PxSpecImages, ci_utils.K8sVersion, t)
cluster = ci_utils.UpdateAndValidateStork(cluster, updateParamFunc, ci_utils.PxSpecImages, t)
require.True(t, cluster.Spec.Stork.Enabled, "failed to validate Stork is enabled: expected: true, actual: %v", cluster.Spec.Stork.Enabled)
}
}
Expand Down
4 changes: 2 additions & 2 deletions test/integration_test/utils/storagecluster.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -243,7 +243,7 @@ func UpdateAndValidatePvcController(cluster *corev1.StorageCluster, f func(*core
}

// UpdateAndValidateStork update StorageCluster, validates Stork components only and return latest version of live StorageCluster
func UpdateAndValidateStork(cluster *corev1.StorageCluster, f func(*corev1.StorageCluster) *corev1.StorageCluster, pxSpecImages map[string]string, k8sVersion string, t *testing.T) *corev1.StorageCluster {
func UpdateAndValidateStork(cluster *corev1.StorageCluster, f func(*corev1.StorageCluster) *corev1.StorageCluster, pxSpecImages map[string]string, t *testing.T) *corev1.StorageCluster {
liveCluster, err := operator.Instance().GetStorageCluster(cluster.Name, cluster.Namespace)
require.NoError(t, err)

Expand All @@ -252,7 +252,7 @@ func UpdateAndValidateStork(cluster *corev1.StorageCluster, f func(*corev1.Stora
latestLiveCluster, err := UpdateStorageCluster(newCluster)
require.NoError(t, err)

err = testutil.ValidateStork(pxSpecImages, latestLiveCluster, k8sVersion, DefaultValidateComponentTimeout, DefaultValidateComponentRetryInterval)
err = testutil.ValidateStork(pxSpecImages, latestLiveCluster, DefaultValidateComponentTimeout, DefaultValidateComponentRetryInterval)
require.NoError(t, err)

return latestLiveCluster
Expand Down

0 comments on commit 9b862bc

Please sign in to comment.