[AzCopyV10][Bug] Uploading from top directory on linux fails when sub folder has the same name

cmd/copy.go

@@ -34,6 +34,7 @@ import (
 	"runtime"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	"github.com/Azure/azure-pipeline-go/pipeline"
@@ -1210,13 +1211,22 @@ type CookedCopyCmdArgs struct {
 	// Optional flag that permanently deletes soft deleted blobs
 	permanentDeleteOption common.PermanentDeleteOption
 
+	// defines whether first part has been ordered or not.
+	// 0 means first part is not ordered and 1 means first part is ordered.
+	atomicFirstPartOrdered uint32
+
 	// Optional flag that sets rehydrate priority for rehydration
 	rehydratePriority common.RehydratePriorityType
 
 	// Bitmasked uint checking which properties to transfer
 	propertiesToTransfer common.SetPropertiesFlags
 }
 
+// setFirstPartOrdered sets the value of atomicFirstPartOrdered to 1
+func (cca *CookedCopyCmdArgs) setFirstPartOrdered() {
+	atomic.StoreUint32(&cca.atomicFirstPartOrdered, 1)
+}
+
 func (cca *CookedCopyCmdArgs) isRedirection() bool {
 	switch cca.FromTo {
 	case common.EFromTo.BlobPipe():

cmd/copyEnumeratorHelper.go

@@ -13,10 +13,13 @@ import (
 var EnumerationParallelism = 1
 var EnumerationParallelStatFiles = false
 
+// TODO: potentially remove addTransfer and replace this with scheduleCopyTransfer
 // addTransfer accepts a new transfer, if the threshold is reached, dispatch a job part order.
 func addTransfer(e *common.CopyJobPartOrderRequest, transfer common.CopyTransfer, cca *CookedCopyCmdArgs) error {
-	// Remove the source and destination roots from the path to save space in the plan files
-	transfer.Source = strings.TrimPrefix(transfer.Source, e.SourceRoot.Value)
+	// Remove the destination roots from the path to save space in the plan files
+
+	// TODO: Remove this code because transfer.Source will already have relative path
+	// transfer.Source = strings.TrimPrefix(transfer.Source, e.SourceRoot.Value)
 	transfer.Destination = strings.TrimPrefix(transfer.Destination, e.DestinationRoot.Value)
 
 	// dispatch the transfers once the number reaches NumOfFilesPerDispatchJobPart

cmd/copyEnumeratorHelper_test.go

@@ -41,23 +41,4 @@ func newRemoteRes(url string) common.ResourceString {
 	return r
 }
 
-func (s *copyEnumeratorHelperTestSuite) TestAddTransferPathRootsTrimmed(c *chk.C) {
-	// setup
-	request := common.CopyJobPartOrderRequest{
-		SourceRoot:      newLocalRes("a/b/"),
-		DestinationRoot: newLocalRes("y/z/"),
-	}
-
-	transfer := common.CopyTransfer{
-		Source:      "a/b/c.txt",
-		Destination: "y/z/c.txt",
-	}
-
-	// execute
-	err := addTransfer(&request, transfer, &CookedCopyCmdArgs{})
-
-	// assert
-	c.Assert(err, chk.IsNil)
-	c.Assert(request.Transfers.List[0].Source, chk.Equals, "c.txt")
-	c.Assert(request.Transfers.List[0].Destination, chk.Equals, "c.txt")
-}
+// TODO: delete this file?

cmd/copyEnumeratorInit.go

@@ -31,11 +31,9 @@ type BucketToContainerNameResolver interface {
 func (cca *CookedCopyCmdArgs) initEnumerator(jobPartOrder common.CopyJobPartOrderRequest, ctx context.Context) (*CopyEnumerator, error) {
 	var traverser ResourceTraverser
 
-	srcCredInfo := common.CredentialInfo{}
-	var isPublic bool
-	var err error
+	srcCredInfo, isPublic, err := GetCredentialInfoForLocation(ctx, cca.FromTo.From(), cca.Source.Value, cca.Source.SAS, true, cca.CpkOptions)
 
-	if srcCredInfo, isPublic, err = GetCredentialInfoForLocation(ctx, cca.FromTo.From(), cca.Source.Value, cca.Source.SAS, true, cca.CpkOptions); err != nil {
+	if err != nil {
 		return nil, err
 		// If S2S and source takes OAuthToken as its cred type (OR) source takes anonymous as its cred type, but it's not public and there's no SAS
 	} else if cca.FromTo.IsS2S() &&
@@ -246,6 +244,11 @@ func (cca *CookedCopyCmdArgs) initEnumerator(jobPartOrder common.CopyJobPartOrde
 		jobsAdmin.JobsAdmin.LogToJobLog(message, pipeline.LogInfo)
 	}
 
+	reportFirstPart := func(jobStarted bool) { cca.setFirstPartOrdered() } // need these values for copy transfer processor
+	reportFinalPart := func() { cca.isEnumerationComplete = true }
+
+	transferScheduler := newCopyTransferProcessor(&jobPartOrder, NumOfFilesPerDispatchJobPart, cca.Source, cca.Destination, reportFirstPart, reportFinalPart, cca.s2sPreserveAccessTier, cca.dryrunMode)
+
 	processor := func(object StoredObject) error {
 		// Start by resolving the name and creating the container
 		if object.ContainerName != "" {
@@ -331,7 +334,7 @@ func (cca *CookedCopyCmdArgs) initEnumerator(jobPartOrder common.CopyJobPartOrde
 		}
 
 		if shouldSendToSte {
-			return addTransfer(&jobPartOrder, transfer, cca)
+			return transferScheduler.scheduleCopyTransfer(object)
 		}
 		return nil
 	}

cmd/zt_copy_s2smigration_test.go

@@ -114,7 +114,7 @@ func validateS2STransfersAreScheduled(c *chk.C, srcDirName string, dstDirName st
 		srcRelativeFilePath, _ := url.PathUnescape(transfer.Source)
 		dstRelativeFilePath, _ := url.PathUnescape(transfer.Destination)
 
-		unescapedSrcDir, _ := url.PathUnescape(srcDirName)
+		/*unescapedSrcDir, _ := url.PathUnescape(srcDirName)
 		unescapedDstDir, _ := url.PathUnescape(dstDirName)
 
 		srcRelativeFilePath = strings.Replace(srcRelativeFilePath, unescapedSrcDir, "", 1)
@@ -123,7 +123,7 @@ func validateS2STransfersAreScheduled(c *chk.C, srcDirName string, dstDirName st
 			// Thing we were searching for is bigger than what we are searching in, due to ending end a /
 			// Happens for root dir
 			dstRelativeFilePath = ""
-		}
+		}*/
 
 		if debugMode {
 			fmt.Println("srcRelativeFilePath: ", srcRelativeFilePath)