Merge pull request #6332 from danfengliu/prepare-for-dm-test

Prepare E2E test of data movement feature and other fixes

test/e2e/Makefile

@@ -55,15 +55,15 @@ VELERO_VERSION ?= $(VERSION)
 PLUGINS ?=
 RESTORE_HELPER_IMAGE ?=
 #Released version only
-UPGRADE_FROM_VELERO_VERSION ?= v1.9.6,v1.10.2
+UPGRADE_FROM_VELERO_VERSION ?= v1.10.2,v1.11.0
 # UPGRADE_FROM_VELERO_CLI can has the same format(a list divided by comma) with UPGRADE_FROM_VELERO_VERSION
 # Upgrade tests will be executed sequently according to the list by UPGRADE_FROM_VELERO_VERSION
 # So although length of UPGRADE_FROM_VELERO_CLI list is not equal with UPGRADE_FROM_VELERO_VERSION
 # Script will still read UPGRADE_FROM_VELERO_CLI list to match UPGRADE_FROM_VELERO_VERSION list from beginning
 # to the end, nil string will be set if UPGRADE_FROM_VELERO_CLI is shorter than UPGRADE_FROM_VELERO_VERSION
 UPGRADE_FROM_VELERO_CLI ?=
 
-MIGRATE_FROM_VELERO_VERSION ?= v1.10.2,self
+MIGRATE_FROM_VELERO_VERSION ?= v1.11.0,self
 MIGRATE_FROM_VELERO_CLI ?=
 
 VELERO_NAMESPACE ?= velero

test/e2e/backup/backup.go

@@ -76,13 +76,6 @@ func BackupRestoreTest(useVolumeSnapshots bool) {
 
 	When("kibishii is the sample workload", func() {
 		It("should be successfully backed up and restored to the default BackupStorageLocation", func() {
-			// TODO[High] - remove code block below when vSphere plugin PR #500 is included in release version.
-			//  because restore will be partiallyFailed when DefaultVolumesToFsBackup is set to true during
-			//  Velero installation with default BSL.
-			if veleroCfg.CloudProvider == "vsphere" && !useVolumeSnapshots {
-				Skip("vSphere plugin PR #500 is not included in latest version 1.4.2")
-			}
-
 			if veleroCfg.InstallVelero {
 				if useVolumeSnapshots {
 					//Install node agent also

test/e2e/test/test.go

@@ -86,7 +86,9 @@ func TestFunc(test VeleroBackupRestoreTest) func() {
 			// Using the global velero config which covered the installation for most common cases
 			veleroCfg := e2e.VeleroCfg
 			// TODO: Skip nodeport test until issue https://github.com/kubernetes/kubernetes/issues/114384 fixed
-			if veleroCfg.CloudProvider == "azure" && strings.Contains(test.GetTestCase().CaseBaseName, "nodeport") {
+			// TODO: Although this issue is closed, but it's not fixed.
+			// TODO: After bump up k8s version in AWS pipeline, this issue also apply for AWS pipeline.
+			if (veleroCfg.CloudProvider == "azure" || veleroCfg.CloudProvider == "aws") && strings.Contains(test.GetTestCase().CaseBaseName, "nodeport") {
 				Skip("Skip due to issue https://github.com/kubernetes/kubernetes/issues/114384 on AKS")
 			}
 			if veleroCfg.InstallVelero {

test/e2e/util/velero/install.go

@@ -59,14 +59,17 @@ func VeleroInstall(ctx context.Context, veleroCfg *VeleroConfig) error {
 	fmt.Printf("Velero install %s\n", time.Now().Format("2006-01-02 15:04:05"))
 	if veleroCfg.CloudProvider != "kind" {
 		fmt.Printf("For cloud platforms, object store plugin provider will be set as cloud provider")
-		veleroCfg.ObjectStoreProvider = veleroCfg.CloudProvider
+		// If ObjectStoreProvider is not provided, then using the value same as CloudProvider
+		if veleroCfg.ObjectStoreProvider == "" {
+			veleroCfg.ObjectStoreProvider = veleroCfg.CloudProvider
+		}
 	} else {
 		if veleroCfg.ObjectStoreProvider == "" {
 			return errors.New("No object store provider specified - must be specified when using kind as the cloud provider") // Gotta have an object store provider
 		}
 	}
 
-	providerPluginsTmp, err := getProviderPlugins(ctx, veleroCfg.VeleroCLI, veleroCfg.ObjectStoreProvider, veleroCfg.Plugins, veleroCfg.Features)
+	providerPluginsTmp, err := getProviderPlugins(ctx, veleroCfg.VeleroCLI, veleroCfg.ObjectStoreProvider, veleroCfg.CloudProvider, veleroCfg.Plugins, veleroCfg.Features)
 	if err != nil {
 		return errors.WithMessage(err, "Failed to get provider plugins")
 	}
@@ -80,7 +83,9 @@ func VeleroInstall(ctx context.Context, veleroCfg *VeleroConfig) error {
 		// We overrider the ObjectStoreProvider here for vSphere because we want to use the aws plugin for the
 		// backup, but needed to pick up the provider plugins earlier.  vSphere plugin no longer needs a Volume
 		// Snapshot location specified
-		veleroCfg.ObjectStoreProvider = "aws"
+		if veleroCfg.ObjectStoreProvider == "" {
+			veleroCfg.ObjectStoreProvider = "aws"
+		}
 		if err := configvSpherePlugin(*veleroCfg.ClientToInstallVelero); err != nil {
 			return errors.WithMessagef(err, "Failed to config vsphere plugin")
 		}

test/e2e/util/velero/velero_utils.go

@@ -55,68 +55,26 @@ const RestoreObjectsPrefix = "restores"
 const PluginsObjectsPrefix = "plugins"
 
 var pluginsMatrix = map[string]map[string][]string{
-	"v1.4": {
-		"aws":       {"velero/velero-plugin-for-aws:v1.1.0"},
-		"azure":     {"velero/velero-plugin-for-microsoft-azure:v1.1.2"},
-		"vsphere":   {"velero/velero-plugin-for-aws:v1.1.0", "vsphereveleroplugin/velero-plugin-for-vsphere:v1.0.2"},
-		"gcp":       {"velero/velero-plugin-for-gcp:v1.1.0"},
-		"azure-csi": {"velero/velero-plugin-for-microsoft-azure:v1.1.2", "velero/velero-plugin-for-csi:v0.1.1 "},
-	},
-	"v1.5": {
-		"aws":       {"velero/velero-plugin-for-aws:v1.1.0"},
-		"azure":     {"velero/velero-plugin-for-microsoft-azure:v1.1.2"},
-		"vsphere":   {"velero/velero-plugin-for-aws:v1.1.0", "vsphereveleroplugin/velero-plugin-for-vsphere:v1.1.1"},
-		"gcp":       {"velero/velero-plugin-for-gcp:v1.1.0"},
-		"azure-csi": {"velero/velero-plugin-for-microsoft-azure:v1.1.2", "velero/velero-plugin-for-csi:v0.1.2 "},
-	},
-	"v1.6": {
-		"aws":       {"velero/velero-plugin-for-aws:v1.2.1"},
-		"azure":     {"velero/velero-plugin-for-microsoft-azure:v1.2.1"},
-		"vsphere":   {"velero/velero-plugin-for-aws:v1.2.1", "vsphereveleroplugin/velero-plugin-for-vsphere:v1.1.1"},
-		"gcp":       {"velero/velero-plugin-for-gcp:v1.2.1"},
-		"azure-csi": {"velero/velero-plugin-for-microsoft-azure:v1.3.0", "velero/velero-plugin-for-csi:v0.1.2 "},
-	},
-	"v1.7": {
-		"aws":       {"velero/velero-plugin-for-aws:v1.3.0"},
-		"azure":     {"velero/velero-plugin-for-microsoft-azure:v1.3.0"},
-		"vsphere":   {"velero/velero-plugin-for-aws:v1.3.0", "vsphereveleroplugin/velero-plugin-for-vsphere:v1.3.0"},
-		"gcp":       {"velero/velero-plugin-for-gcp:v1.3.0"},
-		"azure-csi": {"velero/velero-plugin-for-microsoft-azure:v1.3.0", "velero/velero-plugin-for-csi:v0.2.0"},
-	},
-	"v1.8": {
-		"aws":       {"velero/velero-plugin-for-aws:v1.4.0"},
-		"azure":     {"velero/velero-plugin-for-microsoft-azure:v1.4.0"},
-		"vsphere":   {"velero/velero-plugin-for-aws:v1.4.0", "vsphereveleroplugin/velero-plugin-for-vsphere:v1.3.1"},
-		"gcp":       {"velero/velero-plugin-for-gcp:v1.4.0"},
-		"azure-csi": {"velero/velero-plugin-for-microsoft-azure:v1.4.0", "velero/velero-plugin-for-csi:v0.2.0"},
-	},
-	"v1.9": {
-		"aws":       {"velero/velero-plugin-for-aws:v1.5.0"},
-		"azure":     {"velero/velero-plugin-for-microsoft-azure:v1.5.0"},
-		"vsphere":   {"velero/velero-plugin-for-aws:v1.5.0", "vsphereveleroplugin/velero-plugin-for-vsphere:v1.4.0"},
-		"gcp":       {"velero/velero-plugin-for-gcp:v1.5.0"},
-		"azure-csi": {"velero/velero-plugin-for-microsoft-azure:v1.5.0", "velero/velero-plugin-for-csi:v0.3.0"},
-	},
 	"v1.10": {
-		"aws":       {"velero/velero-plugin-for-aws:v1.6.0"},
-		"azure":     {"velero/velero-plugin-for-microsoft-azure:v1.6.0"},
-		"vsphere":   {"velero/velero-plugin-for-aws:v1.6.0", "vsphereveleroplugin/velero-plugin-for-vsphere:v1.4.1"},
-		"gcp":       {"velero/velero-plugin-for-gcp:v1.6.0"},
-		"azure-csi": {"velero/velero-plugin-for-microsoft-azure:v1.6.0", "velero/velero-plugin-for-csi:v0.4.0"},
+		"aws":     {"velero/velero-plugin-for-aws:v1.6.0"},
+		"azure":   {"velero/velero-plugin-for-microsoft-azure:v1.6.0"},
+		"vsphere": {"vsphereveleroplugin/velero-plugin-for-vsphere:v1.5.1"},
+		"gcp":     {"velero/velero-plugin-for-gcp:v1.6.0"},
+		"csi":     {"velero/velero-plugin-for-csi:v0.4.0"},
 	},
 	"v1.11": {
-		"aws":       {"velero/velero-plugin-for-aws:v1.7.0"},
-		"azure":     {"velero/velero-plugin-for-microsoft-azure:v1.7.0"},
-		"vsphere":   {"velero/velero-plugin-for-aws:v1.7.0", "vsphereveleroplugin/velero-plugin-for-vsphere:v1.4.2"},
-		"gcp":       {"velero/velero-plugin-for-gcp:v1.7.0"},
-		"azure-csi": {"velero/velero-plugin-for-microsoft-azure:v1.7.0", "velero/velero-plugin-for-csi:v0.5.0"},
+		"aws":     {"velero/velero-plugin-for-aws:v1.7.0"},
+		"azure":   {"velero/velero-plugin-for-microsoft-azure:v1.7.0"},
+		"vsphere": {"vsphereveleroplugin/velero-plugin-for-vsphere:v1.5.1"},
+		"gcp":     {"velero/velero-plugin-for-gcp:v1.7.0"},
+		"csi":     {"velero/velero-plugin-for-csi:v0.5.0"},
 	},
 	"main": {
-		"aws":       {"velero/velero-plugin-for-aws:main"},
-		"azure":     {"velero/velero-plugin-for-microsoft-azure:main"},
-		"vsphere":   {"velero/velero-plugin-for-aws:main", "vsphereveleroplugin/velero-plugin-for-vsphere:v1.3.1"},
-		"gcp":       {"velero/velero-plugin-for-gcp:main"},
-		"azure-csi": {"velero/velero-plugin-for-microsoft-azure:main", "velero/velero-plugin-for-csi:main"},
+		"aws":     {"velero/velero-plugin-for-aws:main"},
+		"azure":   {"velero/velero-plugin-for-microsoft-azure:main"},
+		"vsphere": {"vsphereveleroplugin/velero-plugin-for-vsphere:v1.5.1"},
+		"gcp":     {"velero/velero-plugin-for-gcp:main"},
+		"csi":     {"velero/velero-plugin-for-csi:main"},
 	},
 }
 
@@ -132,14 +90,19 @@ func GetProviderPluginsByVersion(version, providerName, feature string) ([]strin
 			return nil, errors.Errorf("fail to get plugins by version: main")
 		}
 	}
-	if strings.EqualFold(providerName, "azure") && strings.EqualFold(feature, "EnableCSI") {
-		providerName = "azure-csi"
-	}
+	var pluginsForFeature []string
+
 	plugins, ok := cloudMap[providerName]
 	if !ok {
 		return nil, errors.Errorf("fail to get plugins by version: %s and provider %s", version, providerName)
 	}
-	return plugins, nil
+	if strings.EqualFold(feature, "EnableCSI") {
+		pluginsForFeature, ok = cloudMap["csi"]
+		if !ok {
+			return nil, errors.Errorf("fail to get plugins by version: %s and provider %s", version, providerName)
+		}
+	}
+	return append(plugins, pluginsForFeature...), nil
 }
 
 // getProviderVeleroInstallOptions returns Velero InstallOptions for the provider.
@@ -452,8 +415,7 @@ func VeleroScheduleCreate(ctx context.Context, veleroCLI string, veleroNamespace
 }
 
 func VeleroSchedulePause(ctx context.Context, veleroCLI string, veleroNamespace string, scheduleName string) error {
-	var args []string
-	args = []string{
+	args := []string{
 		"--namespace", veleroNamespace, "schedule", "pause", scheduleName,
 	}
 	if err := VeleroCmdExec(ctx, veleroCLI, args); err != nil {
@@ -463,8 +425,7 @@ func VeleroSchedulePause(ctx context.Context, veleroCLI string, veleroNamespace
 }
 
 func VeleroScheduleUnpause(ctx context.Context, veleroCLI string, veleroNamespace string, scheduleName string) error {
-	var args []string
-	args = []string{
+	args := []string{
 		"--namespace", veleroNamespace, "schedule", "unpause", scheduleName,
 	}
 	if err := VeleroCmdExec(ctx, veleroCLI, args); err != nil {
@@ -555,7 +516,7 @@ func VeleroVersion(ctx context.Context, veleroCLI, veleroNamespace string) error
 	return nil
 }
 
-func getProviderPlugins(ctx context.Context, veleroCLI, objectStoreProvider, providerPlugins, feature string) ([]string, error) {
+func getProviderPlugins(ctx context.Context, veleroCLI, objectStoreProvider, cloudProvider, providerPlugins, feature string) ([]string, error) {
 	// Fetch the plugins for the provider before checking for the object store provider below.
 	var plugins []string
 	if len(providerPlugins) > 0 {
@@ -569,14 +530,21 @@ func getProviderPlugins(ctx context.Context, veleroCLI, objectStoreProvider, pro
 		if err != nil {
 			return nil, errors.WithMessagef(err, "Fail to get plugin by provider %s and version %s", objectStoreProvider, version)
 		}
+		if objectStoreProvider != "" && cloudProvider != "kind" && objectStoreProvider != cloudProvider {
+			pluginsTmp, err := GetProviderPluginsByVersion(version, cloudProvider, feature)
+			if err != nil {
+				return nil, errors.WithMessage(err, "failed to get velero version")
+			}
+			plugins = append(plugins, pluginsTmp...)
+		}
 	}
 	return plugins, nil
 }
 
 // VeleroAddPluginsForProvider determines which plugins need to be installed for a provider and
 // installs them in the current Velero installation, skipping over those that are already installed.
 func VeleroAddPluginsForProvider(ctx context.Context, veleroCLI string, veleroNamespace string, provider string, addPlugins, feature string) error {
-	plugins, err := getProviderPlugins(ctx, veleroCLI, provider, addPlugins, feature)
+	plugins, err := getProviderPlugins(ctx, veleroCLI, provider, provider, addPlugins, feature)
 	fmt.Printf("addPlugins cmd =%v\n", addPlugins)
 	fmt.Printf("provider cmd = %v\n", provider)
 	fmt.Printf("plugins cmd = %v\n", plugins)