Merge pull request vmware-tanzu#6484 from kaovilai/unkopia-for-plugins

Move kopia logging from util/logging to remove indirect dependency in velero plugins

changelogs/unreleased/6484-kaovilai

@@ -0,0 +1 @@
+Velero Plugins no longer need kopia indirect dependency in their go.mod

pkg/util/logging/kopia_log.go → pkg/kopia/kopia_log.go

@@ -1,3 +1,5 @@
+package kopia
+
 /*
 Copyright the Velero contributors.
 
@@ -14,8 +16,6 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package logging
-
 import (
 	"context"
 

pkg/util/logging/kopia_log_test.go → pkg/kopia/kopia_log_test.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package logging
+package kopia
 
 import (
 	"testing"

pkg/repository/udmrepo/kopialib/lib_repo.go

@@ -33,8 +33,8 @@ import (
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 
+	"github.com/vmware-tanzu/velero/pkg/kopia"
 	"github.com/vmware-tanzu/velero/pkg/repository/udmrepo"
-	"github.com/vmware-tanzu/velero/pkg/util/logging"
 )
 
 type kopiaRepoService struct {
@@ -91,7 +91,7 @@ func NewKopiaRepoService(logger logrus.FieldLogger) udmrepo.BackupRepoService {
 }
 
 func (ks *kopiaRepoService) Init(ctx context.Context, repoOption udmrepo.RepoOptions, createNew bool) error {
-	repoCtx := logging.SetupKopiaLog(ctx, ks.logger)
+	repoCtx := kopia.SetupKopiaLog(ctx, ks.logger)
 
 	if createNew {
 		if err := CreateBackupRepo(repoCtx, repoOption); err != nil {
@@ -113,7 +113,7 @@ func (ks *kopiaRepoService) Open(ctx context.Context, repoOption udmrepo.RepoOpt
 		return nil, errors.Wrapf(err, "repo config %s doesn't exist", repoConfig)
 	}
 
-	repoCtx := logging.SetupKopiaLog(ctx, ks.logger)
+	repoCtx := kopia.SetupKopiaLog(ctx, ks.logger)
 
 	r, err := openKopiaRepo(repoCtx, repoConfig, repoOption.RepoPassword)
 	if err != nil {
@@ -156,7 +156,7 @@ func (ks *kopiaRepoService) Maintain(ctx context.Context, repoOption udmrepo.Rep
 		return errors.Wrapf(err, "repo config %s doesn't exist", repoConfig)
 	}
 
-	repoCtx := logging.SetupKopiaLog(ctx, ks.logger)
+	repoCtx := kopia.SetupKopiaLog(ctx, ks.logger)
 
 	r, err := openKopiaRepo(repoCtx, repoConfig, repoOption.RepoPassword)
 	if err != nil {
@@ -206,7 +206,7 @@ func (ks *kopiaRepoService) DefaultMaintenanceFrequency() time.Duration {
 }
 
 func (km *kopiaMaintenance) runMaintenance(ctx context.Context, rep repo.DirectRepositoryWriter) error {
-	err := snapshotmaintenance.Run(logging.SetupKopiaLog(ctx, km.logger), rep, km.mode, false, maintenance.SafetyFull)
+	err := snapshotmaintenance.Run(kopia.SetupKopiaLog(ctx, km.logger), rep, km.mode, false, maintenance.SafetyFull)
 	if err != nil {
 		return errors.Wrapf(err, "error to run maintenance under mode %s", km.mode)
 	}
@@ -238,7 +238,7 @@ func (kr *kopiaRepository) OpenObject(ctx context.Context, id udmrepo.ID) (udmre
 		return nil, errors.Wrapf(err, "error to parse object ID from %v", id)
 	}
 
-	reader, err := kr.rawRepo.OpenObject(logging.SetupKopiaLog(ctx, kr.logger), objID)
+	reader, err := kr.rawRepo.OpenObject(kopia.SetupKopiaLog(ctx, kr.logger), objID)
 	if err != nil {
 		return nil, errors.Wrap(err, "error to open object")
 	}
@@ -253,7 +253,7 @@ func (kr *kopiaRepository) GetManifest(ctx context.Context, id udmrepo.ID, mani
 		return errors.New("repo is closed or not open")
 	}
 
-	metadata, err := kr.rawRepo.GetManifest(logging.SetupKopiaLog(ctx, kr.logger), manifest.ID(id), mani.Payload)
+	metadata, err := kr.rawRepo.GetManifest(kopia.SetupKopiaLog(ctx, kr.logger), manifest.ID(id), mani.Payload)
 	if err != nil {
 		return errors.Wrap(err, "error to get manifest")
 	}
@@ -268,7 +268,7 @@ func (kr *kopiaRepository) FindManifests(ctx context.Context, filter udmrepo.Man
 		return nil, errors.New("repo is closed or not open")
 	}
 
-	metadata, err := kr.rawRepo.FindManifests(logging.SetupKopiaLog(ctx, kr.logger), filter.Labels)
+	metadata, err := kr.rawRepo.FindManifests(kopia.SetupKopiaLog(ctx, kr.logger), filter.Labels)
 	if err != nil {
 		return nil, errors.Wrap(err, "error to find manifests")
 	}
@@ -286,7 +286,7 @@ func (kr *kopiaRepository) Time() time.Time {
 
 func (kr *kopiaRepository) Close(ctx context.Context) error {
 	if kr.rawWriter != nil {
-		err := kr.rawWriter.Close(logging.SetupKopiaLog(ctx, kr.logger))
+		err := kr.rawWriter.Close(kopia.SetupKopiaLog(ctx, kr.logger))
 		if err != nil {
 			return errors.Wrap(err, "error to close repo writer")
 		}
@@ -295,7 +295,7 @@ func (kr *kopiaRepository) Close(ctx context.Context) error {
 	}
 
 	if kr.rawRepo != nil {
-		err := kr.rawRepo.Close(logging.SetupKopiaLog(ctx, kr.logger))
+		err := kr.rawRepo.Close(kopia.SetupKopiaLog(ctx, kr.logger))
 		if err != nil {
 			return errors.Wrap(err, "error to close repo")
 		}
@@ -311,7 +311,7 @@ func (kr *kopiaRepository) NewObjectWriter(ctx context.Context, opt udmrepo.Obje
 		return nil
 	}
 
-	writer := kr.rawWriter.NewObjectWriter(logging.SetupKopiaLog(ctx, kr.logger), object.WriterOptions{
+	writer := kr.rawWriter.NewObjectWriter(kopia.SetupKopiaLog(ctx, kr.logger), object.WriterOptions{
 		Description: opt.Description,
 		Prefix:      index.IDPrefix(opt.Prefix),
 		AsyncWrites: opt.AsyncWrites,
@@ -332,7 +332,7 @@ func (kr *kopiaRepository) PutManifest(ctx context.Context, manifest udmrepo.Rep
 		return "", errors.New("repo writer is closed or not open")
 	}
 
-	id, err := kr.rawWriter.PutManifest(logging.SetupKopiaLog(ctx, kr.logger), manifest.Metadata.Labels, manifest.Payload)
+	id, err := kr.rawWriter.PutManifest(kopia.SetupKopiaLog(ctx, kr.logger), manifest.Metadata.Labels, manifest.Payload)
 	if err != nil {
 		return "", errors.Wrap(err, "error to put manifest")
 	}
@@ -345,7 +345,7 @@ func (kr *kopiaRepository) DeleteManifest(ctx context.Context, id udmrepo.ID) er
 		return errors.New("repo writer is closed or not open")
 	}
 
-	err := kr.rawWriter.DeleteManifest(logging.SetupKopiaLog(ctx, kr.logger), manifest.ID(id))
+	err := kr.rawWriter.DeleteManifest(kopia.SetupKopiaLog(ctx, kr.logger), manifest.ID(id))
 	if err != nil {
 		return errors.Wrap(err, "error to delete manifest")
 	}
@@ -358,7 +358,7 @@ func (kr *kopiaRepository) Flush(ctx context.Context) error {
 		return errors.New("repo writer is closed or not open")
 	}
 
-	err := kr.rawWriter.Flush(logging.SetupKopiaLog(ctx, kr.logger))
+	err := kr.rawWriter.Flush(kopia.SetupKopiaLog(ctx, kr.logger))
 	if err != nil {
 		return errors.Wrap(err, "error to flush repo")
 	}

pkg/uploader/kopia/snapshot.go

@@ -28,9 +28,9 @@ import (
 
 	"github.com/sirupsen/logrus"
 
+	"github.com/vmware-tanzu/velero/pkg/kopia"
 	"github.com/vmware-tanzu/velero/pkg/repository/udmrepo"
 	"github.com/vmware-tanzu/velero/pkg/uploader"
-	"github.com/vmware-tanzu/velero/pkg/util/logging"
 
 	"github.com/kopia/kopia/fs"
 	"github.com/kopia/kopia/fs/localfs"
@@ -121,7 +121,7 @@ func Backup(ctx context.Context, fsUploader SnapshotUploader, repoWriter repo.Re
 		return nil, false, errors.Wrap(err, "Unable to get local filesystem entry")
 	}
 
-	kopiaCtx := logging.SetupKopiaLog(ctx, log)
+	kopiaCtx := kopia.SetupKopiaLog(ctx, log)
 	snapID, snapshotSize, err := SnapshotSource(kopiaCtx, repoWriter, fsUploader, sourceInfo, rootDir, forceFull, parentSnapshot, tags, log, "Kopia Uploader")
 	if err != nil {
 		return nil, false, err
@@ -306,7 +306,7 @@ func findPreviousSnapshotManifest(ctx context.Context, rep repo.Repository, sour
 func Restore(ctx context.Context, rep repo.RepositoryWriter, progress *Progress, snapshotID, dest string, log logrus.FieldLogger, cancleCh chan struct{}) (int64, int32, error) {
 	log.Info("Start to restore...")
 
-	kopiaCtx := logging.SetupKopiaLog(ctx, log)
+	kopiaCtx := kopia.SetupKopiaLog(ctx, log)
 
 	snapshot, err := snapshot.LoadSnapshot(kopiaCtx, rep, manifest.ID(snapshotID))
 	if err != nil {

test/e2e/upgrade/upgrade.go

@@ -189,6 +189,11 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, veleroCLI2Version VeleroC
 				By("Snapshot should be created in cloud object store", func() {
 					snapshotCheckPoint, err := GetSnapshotCheckPoint(*veleroCfg.ClientToInstallVelero, veleroCfg, 2,
 						upgradeNamespace, backupName, KibishiiPVCNameList)
+					fmt.Println("err========")
+					fmt.Println(err)
+					// TODO-D
+					time.Sleep(10 * time.Hour)
+
 					Expect(err).NotTo(HaveOccurred(), "Fail to get snapshot checkpoint")
 					Expect(SnapshotsShouldBeCreatedInCloud(veleroCfg.CloudProvider,
 						veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket,

test/e2e/util/csi/common.go

@@ -91,6 +91,43 @@ func GetCsiSnapshotHandle(client TestClient, backupName string) ([]string, error
 	}
 	return snapshotHandleList, nil
 }
+func GetCsiSnapshotHandleV1(client TestClient, backupName string) ([]string, error) {
+	_, snapshotClient, err := GetClients()
+	if err != nil {
+		return nil, err
+	}
+	vscList, err1 := snapshotClient.SnapshotV1().VolumeSnapshotContents().List(context.TODO(), metav1.ListOptions{})
+	if err1 != nil {
+		return nil, err
+	}
+
+	var snapshotHandleList []string
+	for _, i := range vscList.Items {
+		if i.Status == nil {
+			fmt.Println("SnapshotHandle Status s nil")
+			continue
+		}
+		if i.Status.SnapshotHandle == nil {
+			fmt.Println("SnapshotHandle is nil")
+			continue
+		}
+
+		if i.Labels == nil {
+			fmt.Println("VolumeSnapshotContents label is nil")
+			continue
+		}
+
+		if i.Labels["velero.io/backup-name"] == backupName {
+			tmp := strings.Split(*i.Status.SnapshotHandle, "/")
+			snapshotHandleList = append(snapshotHandleList, tmp[len(tmp)-1])
+		}
+	}
+
+	if len(snapshotHandleList) == 0 {
+		fmt.Printf("No VolumeSnapshotContent from backup %s", backupName)
+	}
+	return snapshotHandleList, nil
+}
 func GetVolumeSnapshotContentNameByPod(client TestClient, podName, namespace, backupName string) (string, error) {
 	pvcList, err := GetPvcByPodName(context.Background(), namespace, podName)
 	if err != nil {
@@ -128,11 +165,19 @@ func GetVolumeSnapshotContentNameByPod(client TestClient, podName, namespace, ba
 	return "", errors.New(fmt.Sprintf("Fail to get VolumeSnapshotContentName for pod %s under namespace %s", podName, namespace))
 }
 
-func CheckVolumeSnapshotCR(client TestClient, backupName string, expectedCount int) ([]string, error) {
+func CheckVolumeSnapshotCR(client TestClient, backupName string, expectedCount int, apiVersion string) ([]string, error) {
 	var err error
 	var snapshotContentNameList []string
-	if snapshotContentNameList, err = GetCsiSnapshotHandle(client, backupName); err != nil {
-		return nil, errors.Wrap(err, "Fail to get Azure CSI snapshot content")
+	if apiVersion == "v1beta1" {
+		if snapshotContentNameList, err = GetCsiSnapshotHandle(client, backupName); err != nil {
+			return nil, errors.Wrap(err, "Fail to get Azure CSI snapshot content")
+		}
+	} else if apiVersion == "v1" {
+		if snapshotContentNameList, err = GetCsiSnapshotHandleV1(client, backupName); err != nil {
+			return nil, errors.Wrap(err, "Fail to get Azure CSI snapshot content")
+		}
+	} else {
+		return nil, errors.New("API version is invalid")
 	}
 	if len(snapshotContentNameList) != expectedCount {
 		return nil, errors.New(fmt.Sprintf("Snapshot count %d is not as expect %d", len(snapshotContentNameList), expectedCount))

test/e2e/util/velero/velero_utils.go

@@ -1107,13 +1107,23 @@ func GetResticRepositories(ctx context.Context, veleroNamespace, targetNamespace
 
 func GetSnapshotCheckPoint(client TestClient, VeleroCfg VeleroConfig, expectCount int, namespaceBackedUp, backupName string, KibishiiPVCNameList []string) (SnapshotCheckPoint, error) {
 	var snapshotCheckPoint SnapshotCheckPoint
-	var err error
+
 	snapshotCheckPoint.ExpectCount = expectCount
 	snapshotCheckPoint.NamespaceBackedUp = namespaceBackedUp
 	snapshotCheckPoint.PodName = KibishiiPVCNameList
 	if VeleroCfg.CloudProvider == "azure" && strings.EqualFold(VeleroCfg.Features, "EnableCSI") {
 		snapshotCheckPoint.EnableCSI = true
-		if snapshotCheckPoint.SnapshotIDList, err = util.CheckVolumeSnapshotCR(client, backupName, expectCount); err != nil {
+		resourceName := "snapshot.storage.k8s.io"
+
+		srcVersions, err := GetAPIVersions(VeleroCfg.DefaultClient, resourceName)
+
+		if err != nil {
+			return snapshotCheckPoint, err
+		}
+		if len(srcVersions) == 0 {
+			return snapshotCheckPoint, errors.New("Fail to get APIVersion")
+		}
+		if snapshotCheckPoint.SnapshotIDList, err = util.CheckVolumeSnapshotCR(client, backupName, expectCount, srcVersions[0]); err != nil {
 			return snapshotCheckPoint, errors.Wrapf(err, "Fail to get Azure CSI snapshot content")
 		}
 	}