From 1eb1959de6d294867da8c2eed573911c5325ddbb Mon Sep 17 00:00:00 2001 From: siminsavani-msft <77068571+siminsavani-msft@users.noreply.github.com> Date: Wed, 4 May 2022 20:31:34 -0400 Subject: [PATCH 1/6] Fixing bug where root folder name is removed from path incorrectly --- cmd/copyEnumeratorHelper.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cmd/copyEnumeratorHelper.go b/cmd/copyEnumeratorHelper.go index 83cf6957b..927625a68 100644 --- a/cmd/copyEnumeratorHelper.go +++ b/cmd/copyEnumeratorHelper.go @@ -15,8 +15,10 @@ var EnumerationParallelStatFiles = false // addTransfer accepts a new transfer, if the threshold is reached, dispatch a job part order. func addTransfer(e *common.CopyJobPartOrderRequest, transfer common.CopyTransfer, cca *CookedCopyCmdArgs) error { - // Remove the source and destination roots from the path to save space in the plan files - transfer.Source = strings.TrimPrefix(transfer.Source, e.SourceRoot.Value) + // Remove the destination roots from the path to save space in the plan files + + // TODO: Remove this code because transfer.Source will already have relative path + // transfer.Source = strings.TrimPrefix(transfer.Source, e.SourceRoot.Value) transfer.Destination = strings.TrimPrefix(transfer.Destination, e.DestinationRoot.Value) // dispatch the transfers once the number reaches NumOfFilesPerDispatchJobPart From 5d5070b04cce37cb55b80fc9f786646faa792f2b Mon Sep 17 00:00:00 2001 From: siminsavani-msft <77068571+siminsavani-msft@users.noreply.github.com> Date: Fri, 6 May 2022 12:12:29 -0400 Subject: [PATCH 2/6] Add TODOs and remove test --- cmd/copyEnumeratorHelper.go | 1 + cmd/copyEnumeratorHelper_test.go | 21 +-------------------- 2 files changed, 2 insertions(+), 20 deletions(-) diff --git a/cmd/copyEnumeratorHelper.go b/cmd/copyEnumeratorHelper.go index 927625a68..97a00321b 100644 --- a/cmd/copyEnumeratorHelper.go +++ b/cmd/copyEnumeratorHelper.go @@ -13,6 +13,7 @@ import ( var EnumerationParallelism = 1 var EnumerationParallelStatFiles = false +// TODO: potentially remove addTransfer and replace this with scheduleCopyTransfer // addTransfer accepts a new transfer, if the threshold is reached, dispatch a job part order. func addTransfer(e *common.CopyJobPartOrderRequest, transfer common.CopyTransfer, cca *CookedCopyCmdArgs) error { // Remove the destination roots from the path to save space in the plan files diff --git a/cmd/copyEnumeratorHelper_test.go b/cmd/copyEnumeratorHelper_test.go index ba4920388..bf5a7c1e5 100644 --- a/cmd/copyEnumeratorHelper_test.go +++ b/cmd/copyEnumeratorHelper_test.go @@ -41,23 +41,4 @@ func newRemoteRes(url string) common.ResourceString { return r } -func (s *copyEnumeratorHelperTestSuite) TestAddTransferPathRootsTrimmed(c *chk.C) { - // setup - request := common.CopyJobPartOrderRequest{ - SourceRoot: newLocalRes("a/b/"), - DestinationRoot: newLocalRes("y/z/"), - } - - transfer := common.CopyTransfer{ - Source: "a/b/c.txt", - Destination: "y/z/c.txt", - } - - // execute - err := addTransfer(&request, transfer, &CookedCopyCmdArgs{}) - - // assert - c.Assert(err, chk.IsNil) - c.Assert(request.Transfers.List[0].Source, chk.Equals, "c.txt") - c.Assert(request.Transfers.List[0].Destination, chk.Equals, "c.txt") -} +// TODO: delete this file? From d814ce5768ef03abedbe5f441dbab9cb485b2cb2 Mon Sep 17 00:00:00 2001 From: siminsavani-msft <77068571+siminsavani-msft@users.noreply.github.com> Date: Mon, 27 Jun 2022 16:50:03 -0400 Subject: [PATCH 3/6] replacing addtransfer with schedulecopytransfer --- cmd/copy.go | 10 ++++++++++ cmd/copyEnumeratorInit.go | 14 +++++++++----- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/cmd/copy.go b/cmd/copy.go index a30652378..f99eac445 100644 --- a/cmd/copy.go +++ b/cmd/copy.go @@ -34,6 +34,7 @@ import ( "runtime" "strings" "sync" + "sync/atomic" "time" "github.com/Azure/azure-pipeline-go/pipeline" @@ -1150,6 +1151,15 @@ type CookedCopyCmdArgs struct { // Optional flag that permanently deletes soft deleted blobs permanentDeleteOption common.PermanentDeleteOption + + // defines whether first part has been ordered or not. + // 0 means first part is not ordered and 1 means first part is ordered. + atomicFirstPartOrdered uint32 +} + +// setFirstPartOrdered sets the value of atomicFirstPartOrdered to 1 +func (cca *CookedCopyCmdArgs) setFirstPartOrdered() { + atomic.StoreUint32(&cca.atomicFirstPartOrdered, 1) } func (cca *CookedCopyCmdArgs) isRedirection() bool { diff --git a/cmd/copyEnumeratorInit.go b/cmd/copyEnumeratorInit.go index dc1db7bbf..4d75eeee0 100644 --- a/cmd/copyEnumeratorInit.go +++ b/cmd/copyEnumeratorInit.go @@ -30,11 +30,9 @@ type BucketToContainerNameResolver interface { func (cca *CookedCopyCmdArgs) initEnumerator(jobPartOrder common.CopyJobPartOrderRequest, ctx context.Context) (*CopyEnumerator, error) { var traverser ResourceTraverser - srcCredInfo := common.CredentialInfo{} - var isPublic bool - var err error + srcCredInfo, isPublic, err := GetCredentialInfoForLocation(ctx, cca.FromTo.From(), cca.Source.Value, cca.Source.SAS, true, cca.CpkOptions) - if srcCredInfo, isPublic, err = GetCredentialInfoForLocation(ctx, cca.FromTo.From(), cca.Source.Value, cca.Source.SAS, true, cca.CpkOptions); err != nil { + if err != nil { return nil, err // If S2S and source takes OAuthToken as its cred type (OR) source takes anonymous as its cred type, but it's not public and there's no SAS } else if cca.FromTo.From().IsRemote() && cca.FromTo.To().IsRemote() && @@ -226,6 +224,11 @@ func (cca *CookedCopyCmdArgs) initEnumerator(jobPartOrder common.CopyJobPartOrde jobsAdmin.JobsAdmin.LogToJobLog(message, pipeline.LogInfo) } + reportFirstPart := func(jobStarted bool) { cca.setFirstPartOrdered() } // need these values for copy transfer processor + reportFinalPart := func() { cca.isEnumerationComplete = true } + + transferScheduler := newCopyTransferProcessor(&jobPartOrder, NumOfFilesPerDispatchJobPart, cca.Source, cca.Destination, reportFirstPart, reportFinalPart, cca.s2sPreserveAccessTier, cca.dryrunMode) + processor := func(object StoredObject) error { // Start by resolving the name and creating the container if object.ContainerName != "" { @@ -259,6 +262,7 @@ func (cca *CookedCopyCmdArgs) initEnumerator(jobPartOrder common.CopyJobPartOrde srcRelPath := cca.MakeEscapedRelativePath(true, isDestDir, cca.asSubdir, object) dstRelPath := cca.MakeEscapedRelativePath(false, isDestDir, cca.asSubdir, object) + object.relativePath = srcRelPath // set rel path in stored object transfer, shouldSendToSte := object.ToNewCopyTransfer( cca.autoDecompress && cca.FromTo.IsDownload(), @@ -311,7 +315,7 @@ func (cca *CookedCopyCmdArgs) initEnumerator(jobPartOrder common.CopyJobPartOrde } if shouldSendToSte { - return addTransfer(&jobPartOrder, transfer, cca) + return transferScheduler.scheduleCopyTransfer(object) } return nil } From 3e7c9ce33a077218a6e41c2827ec8b67ac48fb9c Mon Sep 17 00:00:00 2001 From: siminsavani-msft <77068571+siminsavani-msft@users.noreply.github.com> Date: Mon, 27 Jun 2022 16:54:42 -0400 Subject: [PATCH 4/6] update copy.go --- cmd/copy.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cmd/copy.go b/cmd/copy.go index 5f662ed26..5ffd45a61 100644 --- a/cmd/copy.go +++ b/cmd/copy.go @@ -1214,11 +1214,6 @@ type CookedCopyCmdArgs struct { // defines whether first part has been ordered or not. // 0 means first part is not ordered and 1 means first part is ordered. atomicFirstPartOrdered uint32 -} - -// setFirstPartOrdered sets the value of atomicFirstPartOrdered to 1 -func (cca *CookedCopyCmdArgs) setFirstPartOrdered() { - atomic.StoreUint32(&cca.atomicFirstPartOrdered, 1) // Optional flag that sets rehydrate priority for rehydration rehydratePriority common.RehydratePriorityType @@ -1227,6 +1222,11 @@ func (cca *CookedCopyCmdArgs) setFirstPartOrdered() { propertiesToTransfer common.SetPropertiesFlags } +// setFirstPartOrdered sets the value of atomicFirstPartOrdered to 1 +func (cca *CookedCopyCmdArgs) setFirstPartOrdered() { + atomic.StoreUint32(&cca.atomicFirstPartOrdered, 1) +} + func (cca *CookedCopyCmdArgs) isRedirection() bool { switch cca.FromTo { case common.EFromTo.BlobPipe(): From 1e402ead0a5a5d1d8d62ba2d16c20eff0d4c2381 Mon Sep 17 00:00:00 2001 From: siminsavani-msft <77068571+siminsavani-msft@users.noreply.github.com> Date: Fri, 8 Jul 2022 15:06:24 -0400 Subject: [PATCH 5/6] Removing update to object relative path --- cmd/copyEnumeratorInit.go | 1 - 1 file changed, 1 deletion(-) diff --git a/cmd/copyEnumeratorInit.go b/cmd/copyEnumeratorInit.go index 8c0dd3f13..27bd5bfc7 100755 --- a/cmd/copyEnumeratorInit.go +++ b/cmd/copyEnumeratorInit.go @@ -282,7 +282,6 @@ func (cca *CookedCopyCmdArgs) initEnumerator(jobPartOrder common.CopyJobPartOrde srcRelPath := cca.MakeEscapedRelativePath(true, isDestDir, cca.asSubdir, object) dstRelPath := cca.MakeEscapedRelativePath(false, isDestDir, cca.asSubdir, object) - object.relativePath = srcRelPath // set rel path in stored object transfer, shouldSendToSte := object.ToNewCopyTransfer( cca.autoDecompress && cca.FromTo.IsDownload(), From 60230b69467790a7327f2feb6331168db5c32ca4 Mon Sep 17 00:00:00 2001 From: siminsavani-msft <77068571+siminsavani-msft@users.noreply.github.com> Date: Tue, 12 Jul 2022 12:09:35 -0400 Subject: [PATCH 6/6] Commenting part of test for copy refactor --- cmd/zt_copy_s2smigration_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/zt_copy_s2smigration_test.go b/cmd/zt_copy_s2smigration_test.go index 2e90b2b22..2a7ae6692 100644 --- a/cmd/zt_copy_s2smigration_test.go +++ b/cmd/zt_copy_s2smigration_test.go @@ -114,7 +114,7 @@ func validateS2STransfersAreScheduled(c *chk.C, srcDirName string, dstDirName st srcRelativeFilePath, _ := url.PathUnescape(transfer.Source) dstRelativeFilePath, _ := url.PathUnescape(transfer.Destination) - unescapedSrcDir, _ := url.PathUnescape(srcDirName) + /*unescapedSrcDir, _ := url.PathUnescape(srcDirName) unescapedDstDir, _ := url.PathUnescape(dstDirName) srcRelativeFilePath = strings.Replace(srcRelativeFilePath, unescapedSrcDir, "", 1) @@ -123,7 +123,7 @@ func validateS2STransfersAreScheduled(c *chk.C, srcDirName string, dstDirName st // Thing we were searching for is bigger than what we are searching in, due to ending end a / // Happens for root dir dstRelativeFilePath = "" - } + }*/ if debugMode { fmt.Println("srcRelativeFilePath: ", srcRelativeFilePath)