Enhance the restore priorities list to support specifying the low prioritized resources that need to be restored in the last

changelogs/unreleased/5529-ywk253100

@@ -0,0 +1 @@
+Enhance the restore priorities list to support specifying the low prioritized resources that need to be restored in the last

pkg/cmd/server/server.go

@@ -114,7 +114,7 @@ type serverConfig struct {
 	pluginDir, metricsAddress, defaultBackupLocation                        string
 	backupSyncPeriod, podVolumeOperationTimeout, resourceTerminatingTimeout time.Duration
 	defaultBackupTTL, storeValidationFrequency, defaultCSISnapshotTimeout   time.Duration
-	restoreResourcePriorities                                               []string
+	restoreResourcePriorities                                               restore.Priorities
 	defaultVolumeSnapshotLocations                                          map[string]string
 	restoreOnly                                                             bool
 	disabledControllers                                                     []string
@@ -208,7 +208,7 @@ func NewCommand(f client.Factory) *cobra.Command {
 	command.Flags().DurationVar(&config.podVolumeOperationTimeout, "restic-timeout", config.podVolumeOperationTimeout, "How long backups/restores of pod volumes should be allowed to run before timing out.")
 	command.Flags().BoolVar(&config.restoreOnly, "restore-only", config.restoreOnly, "Run in a mode where only restores are allowed; backups, schedules, and garbage-collection are all disabled. DEPRECATED: this flag will be removed in v2.0. Use read-only backup storage locations instead.")
 	command.Flags().StringSliceVar(&config.disabledControllers, "disable-controllers", config.disabledControllers, fmt.Sprintf("List of controllers to disable on startup. Valid values are %s", strings.Join(controller.DisableableControllers, ",")))
-	command.Flags().StringSliceVar(&config.restoreResourcePriorities, "restore-resource-priorities", config.restoreResourcePriorities, "Desired order of resource restores; any resource not in the list will be restored alphabetically after the prioritized resources.")
+	command.Flags().Var(&config.restoreResourcePriorities, "restore-resource-priorities", "Desired order of resource restores, the priority list contains two parts which are split by \"-\" element. The resources before \"-\" element are restored first as high priorities, the resources after \"-\" element are restored last as low priorities, and any resource not in the list will be restored alphabetically between the high and low priorities.")
 	command.Flags().StringVar(&config.defaultBackupLocation, "default-backup-storage-location", config.defaultBackupLocation, "Name of the default backup storage location. DEPRECATED: this flag will be removed in v2.0. Use \"velero backup-location set --default\" instead.")
 	command.Flags().DurationVar(&config.storeValidationFrequency, "store-validation-frequency", config.storeValidationFrequency, "How often to verify if the storage is valid. Optional. Set this to `0s` to disable sync. Default 1 minute.")
 	command.Flags().Var(&volumeSnapshotLocations, "default-volume-snapshot-locations", "List of unique volume providers and default volume snapshot location (provider1:location-01,provider2:location-02,...)")
@@ -467,6 +467,7 @@ func (s *server) veleroResourcesExist() error {
 	return nil
 }
 
+// High priorities:
 // - Custom Resource Definitions come before Custom Resource so that they can be
 //   restored with their corresponding CRD.
 // - Namespaces go second because all namespaced resources depend on them.
@@ -489,28 +490,36 @@ func (s *server) veleroResourcesExist() error {
 // - CAPI Clusters come before ClusterResourceSets because failing to do so means the CAPI controller-manager will panic.
 //	 Both Clusters and ClusterResourceSets need to come before ClusterResourceSetBinding in order to properly restore workload clusters.
 //   See https://github.com/kubernetes-sigs/cluster-api/issues/4105
-var defaultRestorePriorities = []string{
-	"customresourcedefinitions",
-	"namespaces",
-	"storageclasses",
-	"volumesnapshotclass.snapshot.storage.k8s.io",
-	"volumesnapshotcontents.snapshot.storage.k8s.io",
-	"volumesnapshots.snapshot.storage.k8s.io",
-	"persistentvolumes",
-	"persistentvolumeclaims",
-	"secrets",
-	"configmaps",
-	"serviceaccounts",
-	"limitranges",
-	"pods",
-	// we fully qualify replicasets.apps because prior to Kubernetes 1.16, replicasets also
-	// existed in the extensions API group, but we back up replicasets from "apps" so we want
-	// to ensure that we prioritize restoring from "apps" too, since this is how they're stored
-	// in the backup.
-	"replicasets.apps",
-	"clusterclasses.cluster.x-k8s.io",
-	"clusters.cluster.x-k8s.io",
-	"clusterresourcesets.addons.cluster.x-k8s.io",
+//
+// Low priorities:
+// - Tanzu ClusterBootstrap go last as it can reference any other kind of resources
+var defaultRestorePriorities = restore.Priorities{
+	HighPriorities: []string{
+		"customresourcedefinitions",
+		"namespaces",
+		"storageclasses",
+		"volumesnapshotclass.snapshot.storage.k8s.io",
+		"volumesnapshotcontents.snapshot.storage.k8s.io",
+		"volumesnapshots.snapshot.storage.k8s.io",
+		"persistentvolumes",
+		"persistentvolumeclaims",
+		"secrets",
+		"configmaps",
+		"serviceaccounts",
+		"limitranges",
+		"pods",
+		// we fully qualify replicasets.apps because prior to Kubernetes 1.16, replicasets also
+		// existed in the extensions API group, but we back up replicasets from "apps" so we want
+		// to ensure that we prioritize restoring from "apps" too, since this is how they're stored
+		// in the backup.
+		"replicasets.apps",
+		"clusterclasses.cluster.x-k8s.io",
+		"clusters.cluster.x-k8s.io",
+		"clusterresourcesets.addons.cluster.x-k8s.io",
+	},
+	LowPriorities: []string{
+		"clusterbootstraps.run.tanzu.vmware.com",
+	},
 }
 
 func (s *server) initRestic() error {

pkg/restore/priority.go

@@ -0,0 +1,92 @@
+/*
+Copyright The Velero Contributors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package restore
+
+import (
+	"fmt"
+	"strings"
+)
+
+const (
+	prioritySeparator = "-"
+)
+
+// Priorities defines the desired order of resource operations:
+//   Resources in the HighPriorities list will be handled first
+//   Resources in the LowPriorities list will be handled last
+//   Other resources will be handled alphabetically after the high prioritized resources and before the low prioritized resources
+type Priorities struct {
+	HighPriorities []string
+	LowPriorities  []string
+}
+
+// String returns a string representation of Priority.
+func (p *Priorities) String() string {
+	priorities := p.HighPriorities
+	if len(p.LowPriorities) > 0 {
+		priorities = append(priorities, prioritySeparator)
+		priorities = append(priorities, p.LowPriorities...)
+	}
+	return strings.Join(priorities, ",")
+}
+
+// Set parses the provided string to the priority object
+func (p *Priorities) Set(s string) error {
+	if len(s) == 0 {
+		return nil
+	}
+	strs := strings.Split(s, ",")
+	separatorIndex := -1
+	for i, str := range strs {
+		if str == prioritySeparator {
+			if separatorIndex > -1 {
+				return fmt.Errorf("multiple priority separator %q found", prioritySeparator)
+			}
+			separatorIndex = i
+		}
+	}
+	// has no separator
+	if separatorIndex == -1 {
+		p.HighPriorities = strs
+		return nil
+	}
+	// start with separator
+	if separatorIndex == 0 {
+		// contain only separator
+		if len(strs) == 1 {
+			return nil
+		}
+		p.LowPriorities = strs[1:]
+		return nil
+	}
+	// end with separator
+	if separatorIndex == len(strs)-1 {
+		p.HighPriorities = strs[:len(strs)-1]
+		return nil
+	}
+
+	// separator in the middle
+	p.HighPriorities = strs[:separatorIndex]
+	p.LowPriorities = strs[separatorIndex+1:]
+
+	return nil
+}
+
+// Type specifies the flag type
+func (p *Priorities) Type() string {
+	return "stringArray"
+}

pkg/restore/priority_test.go

@@ -0,0 +1,110 @@
+/*
+Copyright The Velero Contributors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package restore
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestStringOfPriorities(t *testing.T) {
+	priority := Priorities{
+		HighPriorities: []string{"high"},
+	}
+	assert.Equal(t, "high", priority.String())
+
+	priority = Priorities{
+		HighPriorities: []string{"high"},
+		LowPriorities:  []string{"low"},
+	}
+	assert.Equal(t, "high,-,low", priority.String())
+}
+
+func TestSetOfPriority(t *testing.T) {
+	cases := []struct {
+		name       string
+		input      string
+		priorities Priorities
+		hasErr     bool
+	}{
+		{
+			name:       "empty input",
+			input:      "",
+			priorities: Priorities{},
+			hasErr:     false,
+		},
+		{
+			name:  "only high priorities",
+			input: "p0",
+			priorities: Priorities{
+				HighPriorities: []string{"p0"},
+			},
+			hasErr: false,
+		},
+		{
+			name:  "only low priorities",
+			input: "-,p9",
+			priorities: Priorities{
+				LowPriorities: []string{"p9"},
+			},
+			hasErr: false,
+		},
+		{
+			name:       "only separator",
+			input:      "-",
+			priorities: Priorities{},
+			hasErr:     false,
+		},
+		{
+			name:       "multiple separators",
+			input:      "-,-",
+			priorities: Priorities{},
+			hasErr:     true,
+		},
+		{
+			name:  "contain both high and low priorities",
+			input: "p0,p1,p2,-,p9",
+			priorities: Priorities{
+				HighPriorities: []string{"p0", "p1", "p2"},
+				LowPriorities:  []string{"p9"},
+			},
+			hasErr: false,
+		},
+		{
+			name:  "end with separator",
+			input: "p0,-",
+			priorities: Priorities{
+				HighPriorities: []string{"p0"},
+			},
+			hasErr: false,
+		},
+	}
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			p := Priorities{}
+			err := p.Set(c.input)
+			if c.hasErr {
+				require.NotNil(t, err)
+			} else {
+				require.Nil(t, err)
+			}
+			assert.Equal(t, c.priorities, p)
+		})
+	}
+}

pkg/restore/restore.go

@@ -107,7 +107,7 @@ type kubernetesRestorer struct {
 	resticRestorerFactory      restic.RestorerFactory
 	resticTimeout              time.Duration
 	resourceTerminatingTimeout time.Duration
-	resourcePriorities         []string
+	resourcePriorities         Priorities
 	fileSystem                 filesystem.Interface
 	pvRenamer                  func(string) (string, error)
 	logger                     logrus.FieldLogger
@@ -120,7 +120,7 @@ func NewKubernetesRestorer(
 	restoreClient velerov1client.RestoresGetter,
 	discoveryHelper discovery.Helper,
 	dynamicFactory client.DynamicFactory,
-	resourcePriorities []string,
+	resourcePriorities Priorities,
 	namespaceClient corev1.NamespaceInterface,
 	resticRestorerFactory restic.RestorerFactory,
 	resticTimeout time.Duration,
@@ -351,7 +351,7 @@ type restoreContext struct {
 	renamedPVs                     map[string]string
 	pvRenamer                      func(string) (string, error)
 	discoveryHelper                discovery.Helper
-	resourcePriorities             []string
+	resourcePriorities             Priorities
 	hooksWaitGroup                 sync.WaitGroup
 	hooksErrs                      chan error
 	resourceRestoreHooks           []hook.ResourceRestoreHook
@@ -367,19 +367,31 @@ type resourceClientKey struct {
 
 // getOrderedResources returns an ordered list of resource identifiers to restore,
 // based on the provided resource priorities and backup contents. The returned list
-// begins with all of the prioritized resources (in order), and appends to that
-// an alphabetized list of all resources in the backup.
-func getOrderedResources(resourcePriorities []string, backupResources map[string]*archive.ResourceItems) []string {
-	// alphabetize resources in the backup
-	orderedBackupResources := make([]string, 0, len(backupResources))
+// begins with all of the high prioritized resources (in order), ends with all of
+// the low prioritized resources(in order), and an alphabetized list of resources
+// in the backup(pick out the prioritized resources) is put in the middle.
+func getOrderedResources(resourcePriorities Priorities, backupResources map[string]*archive.ResourceItems) []string {
+	priorities := map[string]struct{}{}
+	for _, priority := range resourcePriorities.HighPriorities {
+		priorities[priority] = struct{}{}
+	}
+	for _, priority := range resourcePriorities.LowPriorities {
+		priorities[priority] = struct{}{}
+	}
+
+	// pick the prioritized resources out
+	var orderedBackupResources []string
 	for resource := range backupResources {
+		if _, exist := priorities[resource]; exist {
+			continue
+		}
 		orderedBackupResources = append(orderedBackupResources, resource)
 	}
+	// alphabetize resources in the backup
 	sort.Strings(orderedBackupResources)
 
-	// Main list: everything in resource priorities, followed by what's in the
-	// backup (alphabetized).
-	return append(resourcePriorities, orderedBackupResources...)
+	list := append(resourcePriorities.HighPriorities, orderedBackupResources...)
+	return append(list, resourcePriorities.LowPriorities...)
 }
 
 type progressUpdate struct {
@@ -466,7 +478,7 @@ func (ctx *restoreContext) execute() (Result, Result) {
 		backupResources,
 		make([]restoreableResource, 0),
 		sets.NewString(),
-		[]string{"customresourcedefinitions"},
+		Priorities{HighPriorities: []string{"customresourcedefinitions"}},
 		false,
 	)
 	warnings.Merge(&w)
@@ -1790,7 +1802,7 @@ func (ctx *restoreContext) getOrderedResourceCollection(
 	backupResources map[string]*archive.ResourceItems,
 	restoreResourceCollection []restoreableResource,
 	processedResources sets.String,
-	resourcePriorities []string,
+	resourcePriorities Priorities,
 	includeAllResources bool,
 ) ([]restoreableResource, sets.String, Result, Result) {
 	var warnings, errs Result
@@ -1812,7 +1824,7 @@ func (ctx *restoreContext) getOrderedResourceCollection(
 	if includeAllResources {
 		resourceList = getOrderedResources(resourcePriorities, backupResources)
 	} else {
-		resourceList = resourcePriorities
+		resourceList = resourcePriorities.HighPriorities
 	}
 	for _, resource := range resourceList {
 		// try to resolve the resource via discovery to a complete group/version/resource