From e56a649fa7a20acc5d79d207795e8ffc9e3f3bf6 Mon Sep 17 00:00:00 2001 From: Naveen Revanna Date: Wed, 9 Aug 2023 21:39:45 -0700 Subject: [PATCH 1/3] PWX-32409: Adding Upgrade test This test case verifies that btrfs datastores stays on upgrade on a testbed that qualifies for dmthin. TestStorageClusterDMthinUpgrade upgrades PX to 3.1 from 3.0 and 2.13 over two iterations. It'll also verify that btrfs was installed. Signed-off-by: Naveen Revanna --- test/integration_test/basic_dmthin_test.go | 84 ++++++++++++++++++++++ 1 file changed, 84 insertions(+) diff --git a/test/integration_test/basic_dmthin_test.go b/test/integration_test/basic_dmthin_test.go index 4d87bd51a..2ccb1e249 100644 --- a/test/integration_test/basic_dmthin_test.go +++ b/test/integration_test/basic_dmthin_test.go @@ -10,9 +10,12 @@ import ( "testing" "github.com/hashicorp/go-version" + "github.com/libopenstorage/operator/drivers/storage/portworx" corev1 "github.com/libopenstorage/operator/pkg/apis/core/v1" + k8sutil "github.com/libopenstorage/operator/pkg/util/k8s" "github.com/libopenstorage/operator/test/integration_test/types" ci_utils "github.com/libopenstorage/operator/test/integration_test/utils" + "github.com/portworx/sched-ops/k8s/operator" "github.com/sirupsen/logrus" "github.com/stretchr/testify/require" v1 "k8s.io/api/core/v1" @@ -131,6 +134,38 @@ func generateTestCases(opt OptionsArr, idx uint) map[string]types.TestCase { return retVal } +var testDmthinCases = []types.TestCase{ + { + TestName: "BasicUpgradeStorageClusterDmthinWithAllComponents", + TestrailCaseIDs: []string{"C50241"}, + TestSpec: func(t *testing.T) interface{} { + objects, err := ci_utils.ParseSpecs("storagecluster/storagecluster-with-all-components.yaml") + provider := cloud_provider.GetCloudProvider() + require.NoError(t, err) + cluster, ok := objects[0].(*corev1.StorageCluster) + require.True(t, ok) + cluster.Name = "test-stc" + cluster.Namespace = "kube-system" + cloudSpec := &corev1.CloudStorageSpec{} + cloudSpec.DeviceSpecs = provider.GetDefaultDataDrives() + cloudSpec.SystemMdDeviceSpec = provider.GetDefaultMetadataDrive() + cluster.Spec.CloudStorage = cloudSpec + cluster.Spec.Monitoring.Prometheus.AlertManager.Enabled = false + return cluster + }, + ShouldSkip: func(tc *types.TestCase) bool { + if len(ci_utils.PxUpgradeHopsURLList) == 0 { + logrus.Info("--px-upgrade-hops-url-list is empty, cannot run BasicUpgradeStorageClusterDmthinWithAllComponents test") + return true + } + k8sVersion, _ := k8sutil.GetVersion() + pxVersion := ci_utils.GetPxVersionFromSpecGenURL(ci_utils.PxUpgradeHopsURLList[0]) + return k8sVersion.GreaterThanOrEqual(k8sutil.K8sVer1_22) && pxVersion.LessThan(pxVer2_9) + }, + TestFunc: BasicUpgradeStorageClusterDmthin, + }, +} + func TestStorageClusterDmthinWithoutPxStoreV2Option(t *testing.T) { opt := OptionsArr{} basicInstallTestCases := generateTestCases(opt, 1) @@ -148,6 +183,12 @@ func TestStorageClusterDmthinWithPxStoreV2Option(t *testing.T) { } } +func TestStorageClusterDMthinUpgrade(t *testing.T) { + for _, testCase := range testDmthinCases { + testCase.RunTest(t) + } +} + func getTotalNodes(t *testing.T, cluster *corev1.StorageCluster) uint { allNodes, err := test.GetExpectedPxNodeList(cluster) require.NoError(t, err, "Could not get expected px node list") @@ -290,3 +331,46 @@ func BasicInstallDmthin(tc *types.TestCase) func(*testing.T) { UninstallPX(t, cluster) } } + +// PxUpgradeHopsURLList is interpreted differently in this test as compared to the original intent +// Each member of the hop list serves as the base PX version to upgrade from to the latest release version. +func BasicUpgradeStorageClusterDmthin(tc *types.TestCase) func(*testing.T) { + return func(t *testing.T) { + for _, hopURL := range ci_utils.PxUpgradeHopsURLList { + pxVersion := ci_utils.GetPxVersionFromSpecGenURL(ci_utils.PxUpgradeHopsURLList[0]) + lessThanOrEqualV3_0_0 := pxVersion.LessThanOrEqual(pxVer3_0_0) + require.True(t, lessThanOrEqualV3_0_0, "Update this test for hop URL > 3.0.0") + // Get versions from URL + specImages, err := test.GetImagesFromVersionURL(hopURL, ci_utils.K8sVersion) + require.NoError(t, err) + + logrus.Infof("Installing PX version %v", specImages) + cluster := installAndValidate(t, tc, hopURL, specImages) + pxStoreV2 := isPxStoreV2(t, cluster) + require.False(t, pxStoreV2) + + // Get live StorageCluster + cluster, err = operator.Instance().GetStorageCluster(cluster.Name, cluster.Namespace) + require.NoError(t, err) + + logrus.Infof("Upgrading PX to %v", ci_utils.PxSpecImages) + err = ci_utils.ConstructStorageCluster(cluster, ci_utils.PxSpecGenURL, ci_utils.PxSpecImages) + require.NoError(t, err) + + // Set defaults + k8sVersion, _ := version.NewVersion(ci_utils.K8sVersion) + portworx.SetPortworxDefaults(cluster, k8sVersion) + + // Update live StorageCluster + cluster, err = ci_utils.UpdateStorageCluster(cluster) + require.NoError(t, err) + logrus.Infof("Validate upgraded StorageCluster %s", cluster.Name) + err = test.ValidateStorageCluster(ci_utils.PxSpecImages, cluster, ci_utils.DefaultValidateUpgradeTimeout, ci_utils.DefaultValidateUpgradeRetryInterval, true, "") + require.NoError(t, err) + + pxStoreV2 = isPxStoreV2(t, cluster) + require.False(t, pxStoreV2) + UninstallPX(t, cluster) + } + } +} From bf988180a1e7d36dc299c6fb3963040f8a91e14e Mon Sep 17 00:00:00 2001 From: Naveen Revanna Date: Tue, 15 Aug 2023 21:59:39 -0700 Subject: [PATCH 2/3] Review comments --- test/integration_test/utils/utils.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration_test/utils/utils.go b/test/integration_test/utils/utils.go index c60ffb76b..04b633b27 100644 --- a/test/integration_test/utils/utils.go +++ b/test/integration_test/utils/utils.go @@ -107,7 +107,7 @@ func RunPxCmdRetry(command ...string) (string, string, error) { var err error for i := 0; i < 4; i++ { out, stderr, err = RunPxCmd(command...) - // Occassionally, commands are terminated due to bad connections. Let's retry them. + // Occasionally, commands are terminated due to bad connections. Let's retry them. if err == nil || strings.Contains(err.Error(), "command terminated") { break } From bea122cfca0a8242a530353cb66cddd92c561002 Mon Sep 17 00:00:00 2001 From: Naveen Revanna Date: Wed, 16 Aug 2023 12:05:22 -0700 Subject: [PATCH 3/3] Review comments --- test/integration_test/basic_dmthin_test.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/test/integration_test/basic_dmthin_test.go b/test/integration_test/basic_dmthin_test.go index 2ccb1e249..137cf07f4 100644 --- a/test/integration_test/basic_dmthin_test.go +++ b/test/integration_test/basic_dmthin_test.go @@ -160,9 +160,9 @@ var testDmthinCases = []types.TestCase{ } k8sVersion, _ := k8sutil.GetVersion() pxVersion := ci_utils.GetPxVersionFromSpecGenURL(ci_utils.PxUpgradeHopsURLList[0]) - return k8sVersion.GreaterThanOrEqual(k8sutil.K8sVer1_22) && pxVersion.LessThan(pxVer2_9) + return k8sVersion.GreaterThanOrEqual(k8sutil.K8sVer1_22) && pxVersion.LessThan(pxVer3_0_0) }, - TestFunc: BasicUpgradeStorageClusterDmthin, + TestFunc: BasicUpgradeStorageClusterNoDmthin, }, } @@ -334,7 +334,9 @@ func BasicInstallDmthin(tc *types.TestCase) func(*testing.T) { // PxUpgradeHopsURLList is interpreted differently in this test as compared to the original intent // Each member of the hop list serves as the base PX version to upgrade from to the latest release version. -func BasicUpgradeStorageClusterDmthin(tc *types.TestCase) func(*testing.T) { +// The intention of this test will be to upgrade PX from a version where dmthin is not supported and expect that he upgrade +// continues to have btrfs and not create dmthin datastores. +func BasicUpgradeStorageClusterNoDmthin(tc *types.TestCase) func(*testing.T) { return func(t *testing.T) { for _, hopURL := range ci_utils.PxUpgradeHopsURLList { pxVersion := ci_utils.GetPxVersionFromSpecGenURL(ci_utils.PxUpgradeHopsURLList[0]) @@ -368,6 +370,7 @@ func BasicUpgradeStorageClusterDmthin(tc *types.TestCase) func(*testing.T) { err = test.ValidateStorageCluster(ci_utils.PxSpecImages, cluster, ci_utils.DefaultValidateUpgradeTimeout, ci_utils.DefaultValidateUpgradeRetryInterval, true, "") require.NoError(t, err) + // After upgrade, PX should continue to have btrfs pxStoreV2 = isPxStoreV2(t, cluster) require.False(t, pxStoreV2) UninstallPX(t, cluster)