Refactor code

Signed-off-by: Emruz Hossain <emruz@appscode.com>

Author: Emruz Hossain

go.mod

@@ -6,20 +6,19 @@ require (
 	github.com/spf13/cobra v1.2.1
 	go.bytebuilders.dev/license-verifier/kubernetes v0.9.7
 	gomodules.xyz/flags v0.1.3
-	gomodules.xyz/go-sh v0.1.0
 	gomodules.xyz/logs v0.0.6
 	gomodules.xyz/sets v0.2.1
 	gomodules.xyz/x v0.0.13
 	k8s.io/api v0.21.1
 	k8s.io/apimachinery v0.21.1
 	k8s.io/client-go v0.21.1
 	k8s.io/klog/v2 v2.9.0
-	kmodules.xyz/client-go v0.0.0-20220308080632-2543b61b64fd
-	kmodules.xyz/custom-resources v0.0.0-20220317043135-5c76c52c975d
-	kmodules.xyz/offshoot-api v0.0.0-20220317044008-01567138fc2f
-	sigs.k8s.io/yaml v1.2.0
-	stash.appscode.dev/apimachinery v0.18.1-0.20220321055929-ee132c17707b
-	stash.appscode.dev/elasticsearch v0.0.0-20220317210913-fc6cd8252e2a
+	kmodules.xyz/client-go v0.0.0-20220317213815-2a6d5a5784f2
+	kmodules.xyz/custom-resources v0.0.0-20220317220154-7beb809b1f5e
+	kmodules.xyz/offshoot-api v0.0.0-20220329041708-c076b2bcb0f8
+	sigs.k8s.io/yaml v1.3.0
+	stash.appscode.dev/apimachinery v0.19.1-0.20220407092220-1fb00845d1f7
+	stash.appscode.dev/elasticsearch v0.0.0-20220329183444-6b505bc9eb9f
 )
 
 replace bitbucket.org/ww/goautoneg => gomodules.xyz/goautoneg v0.0.0-20120707110453-a547fc61f48d

go.sum

@@ -2,24 +2,22 @@ package manager
 
 import (
 	"context"
-	kerr "k8s.io/apimachinery/pkg/api/errors"
+	"path/filepath"
+
+	"stash.appscode.dev/apimachinery/apis/stash/v1beta1"
+	"stash.appscode.dev/manifest-backup/pkg/sanitizers"
+
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime/schema"
-	"k8s.io/apimachinery/pkg/runtime/serializer"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/discovery"
 	"k8s.io/client-go/dynamic"
-	"k8s.io/client-go/kubernetes/scheme"
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/restmapper"
-	"path/filepath"
-	"stash.appscode.dev/apimachinery/apis/stash/v1beta1"
-	"stash.appscode.dev/manifest-backup/pkg/manager/sanitizers"
 )
 
 type applicationBackupManager struct {
-	disc              discovery.DiscoveryInterface
 	di                dynamic.Interface
 	namespace         string
 	storage           Writer
@@ -45,175 +43,129 @@ func newApplicationBackupManager(opt BackupOptions) BackupManager {
 }
 
 func (opt applicationBackupManager) Dump() error {
-	var err error
-	opt.config.QPS = 1e6
-	opt.config.Burst = 1e6
-	if err := rest.SetKubernetesDefaults(opt.config); err != nil {
-		return err
-	}
-	opt.config.NegotiatedSerializer = serializer.WithoutConversionCodecFactory{CodecFactory: scheme.Codecs}
-	if opt.config.UserAgent == "" {
-		opt.config.UserAgent = rest.DefaultKubernetesUserAgent()
-	}
-
-	opt.disc, err = discovery.NewDiscoveryClientForConfig(opt.config)
-	if err != nil {
-		return err
-	}
-
-	opt.di, err = dynamic.NewForConfig(opt.config)
-	if err != nil {
-		return err
-	}
-
-	gv, err := schema.ParseGroupVersion(opt.target.APIVersion)
-	if err != nil {
-		return err
-	}
-	gvk := gv.WithKind(opt.target.Kind)
-
-	apiResources, err := restmapper.GetAPIGroupResources(opt.disc)
+	gvr, err := opt.getRootObjectGVR()
 	if err != nil {
-		return err
+		return nil
 	}
-	mapper := restmapper.NewDiscoveryRESTMapper(apiResources)
-	mapping, err := mapper.RESTMapping(schema.GroupKind{Group: gvk.Group, Kind: gvk.Kind}, gvk.Version)
+	rootObj, err := opt.getRootObject(*gvr)
 	if err != nil {
 		return err
 	}
-	ri := opt.di.Resource(mapping.Resource).Namespace(opt.namespace)
 
-	rootObj, err := ri.Get(context.TODO(), opt.target.Name, metav1.GetOptions{})
-	if err != nil {
-		return err
+	rTree := treeBuilder{
+		resourceTree: make(map[types.UID][]resourceRef),
 	}
-
-	resourceTree := make(map[types.UID][]resourceRef)
 	rootUID := types.UID("root")
-	resourceTree[rootUID] = []resourceRef{
+	rTree.resourceTree[rootUID] = []resourceRef{
 		{
-			gvr:       mapping.Resource,
+			gvr:       *gvr,
 			name:      rootObj.GetName(),
 			namespace: rootObj.GetNamespace(),
+			kind:      rootObj.GetKind(),
 		},
 	}
 
 	if opt.includeDependants {
-		err := opt.generateDependencyTree(resourceTree)
+		err := opt.generateDependencyTree(&rTree)
 		if err != nil {
 			return err
 		}
 	}
-	return opt.dumpResourceTree(resourceTree, rootUID)
+	return opt.dumpResourceTree(rTree.resourceTree, rootUID, opt.dataDir)
 }
 
-type resourceRef struct {
-	gvr       schema.GroupVersionResource
-	name      string
-	namespace string
-}
+func (opt *applicationBackupManager) getRootObjectGVR() (*schema.GroupVersionResource, error) {
+	disc, err := discovery.NewDiscoveryClientForConfig(opt.config)
+	if err != nil {
+		return nil, err
+	}
+	apiResources, err := restmapper.GetAPIGroupResources(disc)
+	if err != nil {
+		return nil, err
+	}
 
-func (opt *applicationBackupManager) generateDependencyTree(resourceTree map[types.UID][]resourceRef) error {
-	resList, err := opt.disc.ServerPreferredResources()
+	gv, err := schema.ParseGroupVersion(opt.target.APIVersion)
 	if err != nil {
-		return err
+		return nil, err
 	}
+	gvk := gv.WithKind(opt.target.Kind)
 
-	for _, group := range resList {
-		err := opt.traverseGroup(resourceTree, group)
-		if err != nil {
-			return err
-		}
+	mapper := restmapper.NewDiscoveryRESTMapper(apiResources)
+	mapping, err := mapper.RESTMapping(schema.GroupKind{Group: gvk.Group, Kind: gvk.Kind}, gvk.Version)
+	if err != nil {
+		return nil, err
 	}
-	return nil
+	return &mapping.Resource, nil
 }
 
-func (opt *applicationBackupManager) traverseGroup(resourceTree map[types.UID][]resourceRef, group *metav1.APIResourceList) error {
-	gv, err := schema.ParseGroupVersion(group.GroupVersion)
+func (opt *applicationBackupManager) getRootObject(gvr schema.GroupVersionResource) (*unstructured.Unstructured, error) {
+	var err error
+	opt.di, err = dynamic.NewForConfig(opt.config)
 	if err != nil {
-		return err
+		return nil, err
 	}
+	ri := opt.di.Resource(gvr).Namespace(opt.namespace)
+	return ri.Get(context.TODO(), opt.target.Name, metav1.GetOptions{})
+}
 
-	for _, res := range group.APIResources {
-		if isSubResource(res.Name) || !hasGetListVerbs(res.Verbs) {
-			continue
-		}
-		// don't process non-namespaced resources when target is a namespace
-		if !res.Namespaced && opt.namespace != "" {
-			continue
-		}
-
-		err := opt.traverseResourceInstances(resourceTree, gv.WithResource(res.Name))
-		if err != nil {
-			return err
-		}
+func (opt *applicationBackupManager) generateDependencyTree(tb *treeBuilder) error {
+	rp := resourceProcessor{
+		config:        opt.config,
+		namespace:     opt.namespace,
+		selector:      opt.selector,
+		itemProcessor: tb,
 	}
-	return nil
+	return rp.processAPIResources()
 }
 
-func (opt *applicationBackupManager) traverseResourceInstances(resourceTree map[types.UID][]resourceRef, gvr schema.GroupVersionResource) error {
-	var next string
-	for {
-		var ri dynamic.ResourceInterface
-		if opt.namespace != "" {
-			ri = opt.di.Resource(gvr).Namespace(opt.namespace)
-		} else {
-			ri = opt.di.Resource(gvr)
-		}
-
-		resp, err := ri.List(context.TODO(), metav1.ListOptions{
-			Limit:         250,
-			Continue:      next,
-			LabelSelector: opt.selector,
-		})
-		if err != nil {
-			if !kerr.IsNotFound(err) {
-				return err
-			}
-			return nil
-		}
-
-		opt.processItems(resourceTree, resp.Items, gvr)
+type resourceRef struct {
+	gvr       schema.GroupVersionResource
+	name      string
+	namespace string
+	kind      string
+}
 
-		next = resp.GetContinue()
-		if next == "" {
-			break
-		}
-	}
-	return nil
+type treeBuilder struct {
+	resourceTree map[types.UID][]resourceRef
 }
 
-func (opt *applicationBackupManager) processItems(resourceTree map[types.UID][]resourceRef, items []unstructured.Unstructured, gvr schema.GroupVersionResource) {
+func (opt treeBuilder) Process(items []unstructured.Unstructured, gvr schema.GroupVersionResource) error {
 	for _, r := range items {
 		ownerRefs := r.GetOwnerReferences()
 		for i := range ownerRefs {
-			if _, ok := resourceTree[ownerRefs[i].UID]; !ok {
-				resourceTree[ownerRefs[i].UID] = make([]resourceRef, 0)
+			if _, ok := opt.resourceTree[ownerRefs[i].UID]; !ok {
+				opt.resourceTree[ownerRefs[i].UID] = make([]resourceRef, 0)
 			}
-			resourceTree[ownerRefs[i].UID] = append(resourceTree[ownerRefs[i].UID], resourceRef{
+			opt.resourceTree[ownerRefs[i].UID] = append(opt.resourceTree[ownerRefs[i].UID], resourceRef{
 				gvr:       gvr,
 				name:      r.GetName(),
 				namespace: r.GetNamespace(),
+				kind:      r.GetKind(),
 			})
 		}
 	}
+	return nil
 }
 
-func (opt *applicationBackupManager) dumpResourceTree(resourceTree map[types.UID][]resourceRef, rootUID types.UID) error {
+func (opt *applicationBackupManager) dumpResourceTree(resourceTree map[types.UID][]resourceRef, rootUID types.UID, prefix string) error {
 	for _, r := range resourceTree[rootUID] {
-		childUID, err := opt.dumpItem(r)
+		childUID, err := opt.dumpItem(r, prefix)
 		if err != nil {
 			return err
 		}
-		err = opt.dumpResourceTree(resourceTree, childUID)
+		childPrefix := prefix
+		if rootUID != "root" {
+			childPrefix = filepath.Join(prefix, r.kind, r.name)
+		}
+		err = opt.dumpResourceTree(resourceTree, childUID, childPrefix)
 		if err != nil {
 			return err
 		}
 	}
 	return nil
 }
 
-func (opt *applicationBackupManager) dumpItem(r resourceRef) (types.UID, error) {
+func (opt *applicationBackupManager) dumpItem(r resourceRef, prefix string) (types.UID, error) {
 	var ri dynamic.ResourceInterface
 	if r.namespace != "" {
 		ri = opt.di.Resource(r.gvr).Namespace(r.namespace)
@@ -237,16 +189,15 @@ func (opt *applicationBackupManager) dumpItem(r resourceRef) (types.UID, error)
 		delete(data, "status")
 	}
 
-	fileName := getFileName(obj, true, opt.dataDir)
+	fileName := opt.getFileName(obj, prefix)
 	return uid, storeItem(fileName, data, opt.storage)
 }
 
-func getFileName(r *unstructured.Unstructured, isNamespaced bool, dataDir string) string {
-	prefix := ""
-	if isNamespaced {
-		prefix = filepath.Join(dataDir, "namespaces", r.GetNamespace())
-	} else {
-		prefix = filepath.Join(dataDir, "global")
+func (opt *applicationBackupManager) getFileName(r *unstructured.Unstructured, prefix string) string {
+	if opt.target.Kind == r.GetKind() &&
+		opt.target.Name == r.GetName() &&
+		opt.namespace == r.GetNamespace() {
+		return filepath.Join(prefix, r.GetName()) + ".yaml"
 	}
-	return filepath.Join(prefix, r.GetKind(), r.GetName()) + ".yaml"
+	return filepath.Join(prefix, r.GetKind(), r.GetName(), r.GetName()) + ".yaml"
 }