From 952d183bd9c1c2430781250e38e1adb462b92cc2 Mon Sep 17 00:00:00 2001 From: Adele Reed Date: Wed, 1 Feb 2023 15:56:13 -0800 Subject: [PATCH 1/6] Implement container-level ACL copies --- cmd/copyEnumeratorInit.go | 4 +-- cmd/zc_traverser_blob.go | 28 +++++++++++++++++- e2etest/declarativeScenario.go | 2 +- e2etest/zt_preserve_properties_test.go | 2 +- ste/sender-blobFolders.go | 40 ++++++++++++++++++++++---- ste/sourceInfoProvider-Blob.go | 8 ++++-- 6 files changed, 72 insertions(+), 12 deletions(-) diff --git a/cmd/copyEnumeratorInit.go b/cmd/copyEnumeratorInit.go index c3541046e..8f0a4881f 100755 --- a/cmd/copyEnumeratorInit.go +++ b/cmd/copyEnumeratorInit.go @@ -117,8 +117,8 @@ func (cca *CookedCopyCmdArgs) initEnumerator(jobPartOrder common.CopyJobPartOrde return nil, errors.New("cannot use --as-subdir=false with a service level destination") } - // When copying a container directly to a container, strip the top directory - if srcLevel == ELocationLevel.Container() && dstLevel == ELocationLevel.Container() && cca.FromTo.From().IsRemote() && cca.FromTo.To().IsRemote() { + // When copying a container directly to a container, strip the top directory, unless we're attempting to persist permissions. + if srcLevel == ELocationLevel.Container() && dstLevel == ELocationLevel.Container() && cca.FromTo.From().IsRemote() && cca.FromTo.To().IsRemote() && !cca.preservePermissions.IsTruthy() { cca.StripTopDir = true } diff --git a/cmd/zc_traverser_blob.go b/cmd/zc_traverser_blob.go index 142de3b58..4b844cbee 100644 --- a/cmd/zc_traverser_blob.go +++ b/cmd/zc_traverser_blob.go @@ -25,6 +25,7 @@ import ( "fmt" "net/url" "strings" + "time" "github.com/Azure/azure-storage-azcopy/v10/common/parallel" @@ -224,7 +225,7 @@ func (t *blobTraverser) Traverse(preprocessor objectMorpher, processor objectPro } } if t.incrementEnumerationCounter != nil { - t.incrementEnumerationCounter(common.EEntityType.File()) + t.incrementEnumerationCounter(storedObject.entityType) } err := processIfPassedFilters(filters, storedObject, processor) @@ -234,6 +235,31 @@ func (t *blobTraverser) Traverse(preprocessor objectMorpher, processor objectPro if !t.includeDeleted && (isBlob || err != nil) { return err } + } else if blobUrlParts.BlobName == "" && t.includeDirectoryStubs { + // if the root is a container and we're copying "folders", we should persist the ACLs there too. + if azcopyScanningLogger != nil { + azcopyScanningLogger.Log(pipeline.LogDebug, "Detected the root as a container.") + } + + storedObject := newStoredObject( + preprocessor, + "", + "", + common.EEntityType.Folder(), + time.Now(), + 0, + noContentProps, + noBlobProps, + common.Metadata{}, + blobUrlParts.ContainerName, + ) + + if t.incrementEnumerationCounter != nil { + t.incrementEnumerationCounter(common.EEntityType.Folder()) + } + + err := processIfPassedFilters(filters, storedObject, processor) + _, err = getProcessingError(err) } // get the container URL so that we can list the blobs diff --git a/e2etest/declarativeScenario.go b/e2etest/declarativeScenario.go index d79f1525f..b911eca02 100644 --- a/e2etest/declarativeScenario.go +++ b/e2etest/declarativeScenario.go @@ -376,7 +376,7 @@ func (s *scenario) getTransferInfo() (srcRoot string, dstRoot string, expectFold // compute dest, taking into account our stripToDir rules addedDirAtDest = "" - areBothContainerLike := s.state.source.isContainerLike() && s.state.dest.isContainerLike() + areBothContainerLike := s.state.source.isContainerLike() && s.state.dest.isContainerLike() && !s.p.preserveSMBPermissions // There are no permission-compatible sources and destinations that do not feature support for root folder perms anymore* tf := s.GetTestFiles() if s.stripTopDir || s.operation == eOperation.Sync() || areBothContainerLike { diff --git a/e2etest/zt_preserve_properties_test.go b/e2etest/zt_preserve_properties_test.go index d94a9514e..a37c66014 100644 --- a/e2etest/zt_preserve_properties_test.go +++ b/e2etest/zt_preserve_properties_test.go @@ -62,7 +62,7 @@ func TestProperties_HNSACLs(t *testing.T) { }, nil, testFiles{ defaultSize: "1K", shouldTransfer: []interface{}{ - folder(""), + folder("", with{adlsPermissionsACL: "user::rwx,group::rw-,other::r--"}), f("filea", with{adlsPermissionsACL: "user::rwx,group::rwx,other::r--"}), folder("a", with{adlsPermissionsACL: "user::rwx,group::rwx,other::-w-"}), f("a/fileb", with{adlsPermissionsACL: "user::rwx,group::rwx,other::--x"}), diff --git a/ste/sender-blobFolders.go b/ste/sender-blobFolders.go index 4b4904055..4402df8d6 100644 --- a/ste/sender-blobFolders.go +++ b/ste/sender-blobFolders.go @@ -76,22 +76,25 @@ func (b *blobFolderSender) setDatalakeACLs() { func (b *blobFolderSender) overwriteDFSProperties() (string, error) { b.jptm.Log(pipeline.LogWarning, "It is impossible to completely overwrite a folder with existing content under it on a hierarchical namespace storage account. A best-effort attempt will be made, but if CPK does not match the transfer will fail.") - b.metadataToApply["hdi_isfolder"] = "true" // Set folder metadata flag err := b.getExtraProperties() if err != nil { return "Get Extra Properties", fmt.Errorf("when getting additional folder properties: %w", err) } + // do not set folder flag as it's invalid to modify a folder with + delete(b.metadataToApply, "hdi_isfolder") + // SetMetadata can set CPK if it wasn't specified prior. This is not a "full" overwrite, but a best-effort overwrite. _, err = b.destination.SetMetadata(b.jptm.Context(), b.metadataToApply, azblob.BlobAccessConditions{}, b.cpkToApply) if err != nil { return "Set Metadata", fmt.Errorf("A best-effort overwrite was attempted; CPK errors cannot be handled when the blob cannot be deleted.\n%w", err) } - _, err = b.destination.SetTags(b.jptm.Context(), nil, nil, nil, b.blobTagsToApply) - if err != nil { - return "Set Blob Tags", err - } + // blob API not yet supported for HNS account error; re-enable later. + //_, err = b.destination.SetTags(b.jptm.Context(), nil, nil, nil, b.blobTagsToApply) + //if err != nil { + // return "Set Blob Tags", err + //} _, err = b.destination.SetHTTPHeaders(b.jptm.Context(), b.headersToAppply, azblob.BlobAccessConditions{}) if err != nil { return "Set HTTP Headers", err @@ -105,9 +108,36 @@ func (b *blobFolderSender) overwriteDFSProperties() (string, error) { return "", nil } +func (b *blobFolderSender) SetContainerACL() error { + bURLParts := azblob.NewBlobURLParts(b.destination.URL()) + bURLParts.BlobName = "/" // Container-level ACLs NEED a / + bURLParts.Host = strings.ReplaceAll(bURLParts.Host, ".blob", ".dfs") + // todo: jank, and violates the principle of interfaces + fileURL := azbfs.NewFileSystemURL(bURLParts.URL(), b.jptm.(*jobPartTransferMgr).jobPartMgr.(*jobPartMgr).secondaryPipeline) + + // We know for a fact our source is a "blob". + acl, err := b.sip.(*blobSourceInfoProvider).AccessControl() + if err != nil { + b.jptm.FailActiveSend("Grabbing source ACLs", err) + return folderPropertiesSetInCreation{} // standard completion will detect failure + } + acl.Permissions = "" // Since we're sending the full ACL, Permissions is irrelevant. + _, err = fileURL.SetAccessControl(b.jptm.Context(), acl) + if err != nil { + b.jptm.FailActiveSend("Putting ACLs", err) + return folderPropertiesSetInCreation{} // standard completion will detect failure + } + + return folderPropertiesSetInCreation{} // standard completion will handle the rest +} + func (b *blobFolderSender) EnsureFolderExists() error { t := b.jptm.GetFolderCreationTracker() + if azblob.NewBlobURLParts(b.destination.URL()).BlobName == "" { + return b.SetContainerACL() // Can't do much with a container, but it is here. + } + _, err := b.destination.GetProperties(b.jptm.Context(), azblob.BlobAccessConditions{}, b.cpkToApply) if err != nil { if stgErr, ok := err.(azblob.StorageError); !(ok && stgErr.ServiceCode() == azblob.ServiceCodeBlobNotFound) { diff --git a/ste/sourceInfoProvider-Blob.go b/ste/sourceInfoProvider-Blob.go index 4e9b3d27a..373530ba4 100644 --- a/ste/sourceInfoProvider-Blob.go +++ b/ste/sourceInfoProvider-Blob.go @@ -79,10 +79,14 @@ func (p *blobSourceInfoProvider) AccessControl() (azbfs.BlobFSAccessControl, err bURLParts := azblob.NewBlobURLParts(*presignedURL) bURLParts.Host = strings.ReplaceAll(bURLParts.Host, ".blob", ".dfs") - bURLParts.BlobName = strings.TrimSuffix(bURLParts.BlobName, "/") // BlobFS doesn't handle folders correctly like this. + if bURLParts.BlobName != "" { + bURLParts.BlobName = strings.TrimSuffix(bURLParts.BlobName, "/") // BlobFS doesn't handle folders correctly like this. + } else { + bURLParts.BlobName = "/" // container level perms MUST have a / + } + // todo: jank, and violates the principle of interfaces fURL := azbfs.NewFileURL(bURLParts.URL(), p.jptm.(*jobPartTransferMgr).jobPartMgr.(*jobPartMgr).secondarySourceProviderPipeline) - return fURL.GetAccessControl(p.jptm.Context()) } From 5be7da6302fa30aa1b1a4e465707b61e91bce2c4 Mon Sep 17 00:00:00 2001 From: Adele Reed Date: Thu, 2 Feb 2023 13:21:51 -0800 Subject: [PATCH 2/6] Prevent AssertNoErr from crashing test suite --- e2etest/declarativeHelpers.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2etest/declarativeHelpers.go b/e2etest/declarativeHelpers.go index ef09d4622..c9f26a674 100644 --- a/e2etest/declarativeHelpers.go +++ b/e2etest/declarativeHelpers.go @@ -116,7 +116,7 @@ func (a *testingAsserter) AssertNoErr(err error, comment ...string) { a.t.Helper() // exclude this method from the logged callstack redactedErr := sanitizer.SanitizeLogMessage(err.Error()) a.t.Logf("Error %s%s", redactedErr, a.formatComments(comment)) - a.t.FailNow() + a.t.Fail() } } From 276771abdc81aad2c63a4a98cf35a981c0c5de21 Mon Sep 17 00:00:00 2001 From: Adele Reed Date: Tue, 7 Feb 2023 10:15:59 -0800 Subject: [PATCH 3/6] Fix delete functionality & testing --- cmd/copyEnumeratorInit.go | 4 ++-- cmd/copyEnumeratorInit_test.go | 10 +++++----- cmd/list.go | 2 +- cmd/removeEnumerator.go | 2 +- cmd/setPropertiesEnumerator.go | 2 +- cmd/syncEnumerator.go | 4 ++-- cmd/zc_enumerator.go | 10 +++++----- cmd/zc_traverser_blob.go | 7 +++++-- cmd/zc_traverser_blob_account.go | 6 ++++-- cmd/zc_traverser_list.go | 4 ++-- cmd/zt_copy_blob_download_test.go | 4 ++-- cmd/zt_generic_service_traverser_test.go | 4 ++-- cmd/zt_generic_traverser_test.go | 10 +++++----- cmd/zt_traverser_blob_test.go | 8 ++++---- e2etest/declarativeScenario.go | 14 ++++++++++++-- 15 files changed, 53 insertions(+), 38 deletions(-) diff --git a/cmd/copyEnumeratorInit.go b/cmd/copyEnumeratorInit.go index 8f0a4881f..00d2410a1 100755 --- a/cmd/copyEnumeratorInit.go +++ b/cmd/copyEnumeratorInit.go @@ -71,7 +71,7 @@ func (cca *CookedCopyCmdArgs) initEnumerator(jobPartOrder common.CopyJobPartOrde traverser, err = InitResourceTraverser(cca.Source, cca.FromTo.From(), &ctx, &srcCredInfo, &cca.FollowSymlinks, cca.ListOfFilesChannel, cca.Recursive, getRemoteProperties, cca.IncludeDirectoryStubs, cca.permanentDeleteOption, func(common.EntityType) {}, cca.ListOfVersionIDs, - cca.S2sPreserveBlobTags, common.ESyncHashType.None(), azcopyLogVerbosity.ToPipelineLogLevel(), cca.CpkOptions, nil /* errorChannel */) + cca.S2sPreserveBlobTags, common.ESyncHashType.None(), cca.preservePermissions, azcopyLogVerbosity.ToPipelineLogLevel(), cca.CpkOptions, nil /* errorChannel */) if err != nil { return nil, err @@ -343,7 +343,7 @@ func (cca *CookedCopyCmdArgs) isDestDirectory(dst common.ResourceString, ctx *co rt, err := InitResourceTraverser(dst, cca.FromTo.To(), ctx, &dstCredInfo, nil, nil, false, false, false, common.EPermanentDeleteOption.None(), - func(common.EntityType) {}, cca.ListOfVersionIDs, false, common.ESyncHashType.None(), pipeline.LogNone, cca.CpkOptions, nil /* errorChannel */) + func(common.EntityType) {}, cca.ListOfVersionIDs, false, common.ESyncHashType.None(), cca.preservePermissions, pipeline.LogNone, cca.CpkOptions, nil /* errorChannel */) if err != nil { return false diff --git a/cmd/copyEnumeratorInit_test.go b/cmd/copyEnumeratorInit_test.go index d99d2e5ca..8a66a088c 100644 --- a/cmd/copyEnumeratorInit_test.go +++ b/cmd/copyEnumeratorInit_test.go @@ -49,7 +49,7 @@ func (ce *copyEnumeratorSuite) TestValidateSourceDirThatExists(c *chk.C) { // List rawBlobURLWithSAS := scenarioHelper{}.getRawBlobURLWithSAS(c, containerName, dirName) - blobTraverser := newBlobTraverser(&rawBlobURLWithSAS, p, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false) + blobTraverser := newBlobTraverser(&rawBlobURLWithSAS, p, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None()) // dir but recursive flag not set - fail cca := CookedCopyCmdArgs{StripTopDir: false, Recursive: false} @@ -78,7 +78,7 @@ func (ce *copyEnumeratorSuite) TestValidateSourceDirDoesNotExist(c *chk.C) { // List rawBlobURLWithSAS := scenarioHelper{}.getRawBlobURLWithSAS(c, containerName, dirName) - blobTraverser := newBlobTraverser(&rawBlobURLWithSAS, p, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false) + blobTraverser := newBlobTraverser(&rawBlobURLWithSAS, p, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None()) // dir but recursive flag not set - fail cca := CookedCopyCmdArgs{StripTopDir: false, Recursive: false} @@ -108,7 +108,7 @@ func (ce *copyEnumeratorSuite) TestValidateSourceFileExists(c *chk.C) { // List rawBlobURLWithSAS := scenarioHelper{}.getRawBlobURLWithSAS(c, containerName, fileName) - blobTraverser := newBlobTraverser(&rawBlobURLWithSAS, p, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false) + blobTraverser := newBlobTraverser(&rawBlobURLWithSAS, p, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None()) cca := CookedCopyCmdArgs{StripTopDir: false, Recursive: false} err := cca.validateSourceDir(blobTraverser) @@ -131,7 +131,7 @@ func (ce *copyEnumeratorSuite) TestValidateSourceFileDoesNotExist(c *chk.C) { // List rawBlobURLWithSAS := scenarioHelper{}.getRawBlobURLWithSAS(c, containerName, fileName) - blobTraverser := newBlobTraverser(&rawBlobURLWithSAS, p, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false) + blobTraverser := newBlobTraverser(&rawBlobURLWithSAS, p, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None()) cca := CookedCopyCmdArgs{StripTopDir: false, Recursive: false} err := cca.validateSourceDir(blobTraverser) @@ -154,7 +154,7 @@ func (ce *copyEnumeratorSuite) TestValidateSourceWithWildCard(c *chk.C) { // List rawBlobURLWithSAS := scenarioHelper{}.getRawBlobURLWithSAS(c, containerName, dirName) - blobTraverser := newBlobTraverser(&rawBlobURLWithSAS, p, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false) + blobTraverser := newBlobTraverser(&rawBlobURLWithSAS, p, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None()) // dir but recursive flag not set - fail cca := CookedCopyCmdArgs{StripTopDir: true, Recursive: false} diff --git a/cmd/list.go b/cmd/list.go index ef2f14b75..3535e859f 100755 --- a/cmd/list.go +++ b/cmd/list.go @@ -224,7 +224,7 @@ func (cooked cookedListCmdArgs) HandleListContainerCommand() (err error) { traverser, err := InitResourceTraverser(source, cooked.location, &ctx, &credentialInfo, nil, nil, true, false, false, common.EPermanentDeleteOption.None(), func(common.EntityType) {}, - nil, false, common.ESyncHashType.None(), pipeline.LogNone, common.CpkOptions{}, nil /* errorChannel */) + nil, false, common.ESyncHashType.None(), common.EPreservePermissionsOption.None(), pipeline.LogNone, common.CpkOptions{}, nil /* errorChannel */) if err != nil { return fmt.Errorf("failed to initialize traverser: %s", err.Error()) diff --git a/cmd/removeEnumerator.go b/cmd/removeEnumerator.go index 71f6e9c85..ecf422220 100755 --- a/cmd/removeEnumerator.go +++ b/cmd/removeEnumerator.go @@ -51,7 +51,7 @@ func newRemoveEnumerator(cca *CookedCopyCmdArgs) (enumerator *CopyEnumerator, er sourceTraverser, err = InitResourceTraverser(cca.Source, cca.FromTo.From(), &ctx, &cca.credentialInfo, nil, cca.ListOfFilesChannel, cca.Recursive, false, cca.IncludeDirectoryStubs, cca.permanentDeleteOption, func(common.EntityType) {}, cca.ListOfVersionIDs, false, - common.ESyncHashType.None(), azcopyLogVerbosity.ToPipelineLogLevel(), cca.CpkOptions, nil /* errorChannel */) + common.ESyncHashType.None(), common.EPreservePermissionsOption.None(), azcopyLogVerbosity.ToPipelineLogLevel(), cca.CpkOptions, nil /* errorChannel */) // report failure to create traverser if err != nil { diff --git a/cmd/setPropertiesEnumerator.go b/cmd/setPropertiesEnumerator.go index d2e35afd9..7663bdd61 100755 --- a/cmd/setPropertiesEnumerator.go +++ b/cmd/setPropertiesEnumerator.go @@ -50,7 +50,7 @@ func setPropertiesEnumerator(cca *CookedCopyCmdArgs) (enumerator *CopyEnumerator sourceTraverser, err = InitResourceTraverser(cca.Source, cca.FromTo.From(), &ctx, &cca.credentialInfo, nil, cca.ListOfFilesChannel, cca.Recursive, false, cca.IncludeDirectoryStubs, cca.permanentDeleteOption, func(common.EntityType) {}, cca.ListOfVersionIDs, false, - common.ESyncHashType.None(), azcopyLogVerbosity.ToPipelineLogLevel(), cca.CpkOptions, nil /* errorChannel */) + common.ESyncHashType.None(), common.EPreservePermissionsOption.None(), azcopyLogVerbosity.ToPipelineLogLevel(), cca.CpkOptions, nil /* errorChannel */) // report failure to create traverser if err != nil { diff --git a/cmd/syncEnumerator.go b/cmd/syncEnumerator.go index 1a8898ad6..5e22ace3b 100644 --- a/cmd/syncEnumerator.go +++ b/cmd/syncEnumerator.go @@ -65,7 +65,7 @@ func (cca *cookedSyncCmdArgs) initEnumerator(ctx context.Context) (enumerator *s if entityType == common.EEntityType.File() { atomic.AddUint64(&cca.atomicSourceFilesScanned, 1) } - }, nil, cca.s2sPreserveBlobTags, cca.compareHash, azcopyLogVerbosity.ToPipelineLogLevel(), cca.cpkOptions, nil /* errorChannel */) + }, nil, cca.s2sPreserveBlobTags, cca.compareHash, cca.preservePermissions, azcopyLogVerbosity.ToPipelineLogLevel(), cca.cpkOptions, nil /* errorChannel */) if err != nil { return nil, err @@ -86,7 +86,7 @@ func (cca *cookedSyncCmdArgs) initEnumerator(ctx context.Context) (enumerator *s if entityType == common.EEntityType.File() { atomic.AddUint64(&cca.atomicDestinationFilesScanned, 1) } - }, nil, cca.s2sPreserveBlobTags, cca.compareHash, azcopyLogVerbosity.ToPipelineLogLevel(), cca.cpkOptions, nil /* errorChannel */) + }, nil, cca.s2sPreserveBlobTags, cca.compareHash, cca.preservePermissions, azcopyLogVerbosity.ToPipelineLogLevel(), cca.cpkOptions, nil /* errorChannel */) if err != nil { return nil, err } diff --git a/cmd/zc_enumerator.go b/cmd/zc_enumerator.go index 3410e6201..7163593bf 100755 --- a/cmd/zc_enumerator.go +++ b/cmd/zc_enumerator.go @@ -334,7 +334,7 @@ type enumerationCounterFunc func(entityType common.EntityType) func InitResourceTraverser(resource common.ResourceString, location common.Location, ctx *context.Context, credential *common.CredentialInfo, followSymlinks *bool, listOfFilesChannel chan string, recursive, getProperties, includeDirectoryStubs bool, permanentDeleteOption common.PermanentDeleteOption, incrementEnumerationCounter enumerationCounterFunc, listOfVersionIds chan string, - s2sPreserveBlobTags bool, syncHashType common.SyncHashType, logLevel pipeline.LogLevel, cpkOptions common.CpkOptions, errorChannel chan ErrorFileInfo) (ResourceTraverser, error) { + s2sPreserveBlobTags bool, syncHashType common.SyncHashType, preservePermissions common.PreservePermissionsOption, logLevel pipeline.LogLevel, cpkOptions common.CpkOptions, errorChannel chan ErrorFileInfo) (ResourceTraverser, error) { var output ResourceTraverser var p *pipeline.Pipeline @@ -387,7 +387,7 @@ func InitResourceTraverser(resource common.ResourceString, location common.Locat } output = newListTraverser(resource, location, credential, ctx, recursive, toFollow, getProperties, - listOfFilesChannel, includeDirectoryStubs, incrementEnumerationCounter, s2sPreserveBlobTags, logLevel, cpkOptions) + listOfFilesChannel, includeDirectoryStubs, incrementEnumerationCounter, s2sPreserveBlobTags, logLevel, cpkOptions, syncHashType, preservePermissions) return output, nil } @@ -415,7 +415,7 @@ func InitResourceTraverser(resource common.ResourceString, location common.Locat baseResource := resource.CloneWithValue(cleanLocalPath(basePath)) output = newListTraverser(baseResource, location, nil, nil, recursive, toFollow, getProperties, - globChan, includeDirectoryStubs, incrementEnumerationCounter, s2sPreserveBlobTags, logLevel, cpkOptions) + globChan, includeDirectoryStubs, incrementEnumerationCounter, s2sPreserveBlobTags, logLevel, cpkOptions, syncHashType, preservePermissions) } else { if ctx != nil { output = newLocalTraverser(*ctx, resource.ValueLocal(), recursive, toFollow, syncHashType, incrementEnumerationCounter, errorChannel) @@ -450,11 +450,11 @@ func InitResourceTraverser(resource common.ResourceString, location common.Locat return nil, errors.New(accountTraversalInherentlyRecursiveError) } - output = newBlobAccountTraverser(resourceURL, *p, *ctx, includeDirectoryStubs, incrementEnumerationCounter, s2sPreserveBlobTags, cpkOptions) + output = newBlobAccountTraverser(resourceURL, *p, *ctx, includeDirectoryStubs, incrementEnumerationCounter, s2sPreserveBlobTags, cpkOptions, preservePermissions) } else if listOfVersionIds != nil { output = newBlobVersionsTraverser(resourceURL, *p, *ctx, recursive, includeDirectoryStubs, incrementEnumerationCounter, listOfVersionIds, cpkOptions) } else { - output = newBlobTraverser(resourceURL, *p, *ctx, recursive, includeDirectoryStubs, incrementEnumerationCounter, s2sPreserveBlobTags, cpkOptions, includeDeleted, includeSnapshot, includeVersion) + output = newBlobTraverser(resourceURL, *p, *ctx, recursive, includeDirectoryStubs, incrementEnumerationCounter, s2sPreserveBlobTags, cpkOptions, includeDeleted, includeSnapshot, includeVersion, preservePermissions) } case common.ELocation.File(): resourceURL, err := resource.FullURL() diff --git a/cmd/zc_traverser_blob.go b/cmd/zc_traverser_blob.go index 4b844cbee..acaf8606c 100644 --- a/cmd/zc_traverser_blob.go +++ b/cmd/zc_traverser_blob.go @@ -57,6 +57,8 @@ type blobTraverser struct { cpkOptions common.CpkOptions + preservePermissions common.PreservePermissionsOption + includeDeleted bool includeSnapshot bool @@ -235,7 +237,7 @@ func (t *blobTraverser) Traverse(preprocessor objectMorpher, processor objectPro if !t.includeDeleted && (isBlob || err != nil) { return err } - } else if blobUrlParts.BlobName == "" && t.includeDirectoryStubs { + } else if blobUrlParts.BlobName == "" && t.preservePermissions.IsTruthy() { // if the root is a container and we're copying "folders", we should persist the ACLs there too. if azcopyScanningLogger != nil { azcopyScanningLogger.Log(pipeline.LogDebug, "Detected the root as a container.") @@ -508,7 +510,7 @@ func (t *blobTraverser) serialList(containerURL azblob.ContainerURL, containerNa return nil } -func newBlobTraverser(rawURL *url.URL, p pipeline.Pipeline, ctx context.Context, recursive, includeDirectoryStubs bool, incrementEnumerationCounter enumerationCounterFunc, s2sPreserveSourceTags bool, cpkOptions common.CpkOptions, includeDeleted, includeSnapshot, includeVersion bool) (t *blobTraverser) { +func newBlobTraverser(rawURL *url.URL, p pipeline.Pipeline, ctx context.Context, recursive, includeDirectoryStubs bool, incrementEnumerationCounter enumerationCounterFunc, s2sPreserveSourceTags bool, cpkOptions common.CpkOptions, includeDeleted, includeSnapshot, includeVersion bool, preservePermissions common.PreservePermissionsOption) (t *blobTraverser) { t = &blobTraverser{ rawURL: rawURL, p: p, @@ -522,6 +524,7 @@ func newBlobTraverser(rawURL *url.URL, p pipeline.Pipeline, ctx context.Context, includeDeleted: includeDeleted, includeSnapshot: includeSnapshot, includeVersion: includeVersion, + preservePermissions: preservePermissions, } disableHierarchicalScanning := strings.ToLower(glcm.GetEnvironmentVariable(common.EEnvironmentVariable.DisableHierarchicalScanning())) diff --git a/cmd/zc_traverser_blob_account.go b/cmd/zc_traverser_blob_account.go index 6c946e01c..b5c3f3980 100644 --- a/cmd/zc_traverser_blob_account.go +++ b/cmd/zc_traverser_blob_account.go @@ -45,6 +45,7 @@ type blobAccountTraverser struct { s2sPreserveSourceTags bool cpkOptions common.CpkOptions + preservePermissions common.PreservePermissionsOption } func (t *blobAccountTraverser) IsDirectory(_ bool) (bool, error) { @@ -100,7 +101,7 @@ func (t *blobAccountTraverser) Traverse(preprocessor objectMorpher, processor ob for _, v := range cList { containerURL := t.accountURL.NewContainerURL(v).URL() - containerTraverser := newBlobTraverser(&containerURL, t.p, t.ctx, true, t.includeDirectoryStubs, t.incrementEnumerationCounter, t.s2sPreserveSourceTags, t.cpkOptions, false, false, false) + containerTraverser := newBlobTraverser(&containerURL, t.p, t.ctx, true, t.includeDirectoryStubs, t.incrementEnumerationCounter, t.s2sPreserveSourceTags, t.cpkOptions, false, false, false, t.preservePermissions) preprocessorForThisChild := preprocessor.FollowedBy(newContainerDecorator(v)) @@ -115,7 +116,7 @@ func (t *blobAccountTraverser) Traverse(preprocessor objectMorpher, processor ob return nil } -func newBlobAccountTraverser(rawURL *url.URL, p pipeline.Pipeline, ctx context.Context, includeDirectoryStubs bool, incrementEnumerationCounter enumerationCounterFunc, s2sPreserveSourceTags bool, cpkOptions common.CpkOptions) (t *blobAccountTraverser) { +func newBlobAccountTraverser(rawURL *url.URL, p pipeline.Pipeline, ctx context.Context, includeDirectoryStubs bool, incrementEnumerationCounter enumerationCounterFunc, s2sPreserveSourceTags bool, cpkOptions common.CpkOptions, preservePermissions common.PreservePermissionsOption) (t *blobAccountTraverser) { bURLParts := azblob.NewBlobURLParts(*rawURL) cPattern := bURLParts.ContainerName @@ -133,6 +134,7 @@ func newBlobAccountTraverser(rawURL *url.URL, p pipeline.Pipeline, ctx context.C includeDirectoryStubs: includeDirectoryStubs, s2sPreserveSourceTags: s2sPreserveSourceTags, cpkOptions: cpkOptions, + preservePermissions: preservePermissions, } return diff --git a/cmd/zc_traverser_list.go b/cmd/zc_traverser_list.go index ec578634e..80d9a82bc 100755 --- a/cmd/zc_traverser_list.go +++ b/cmd/zc_traverser_list.go @@ -92,7 +92,7 @@ func (l *listTraverser) Traverse(preprocessor objectMorpher, processor objectPro func newListTraverser(parent common.ResourceString, parentType common.Location, credential *common.CredentialInfo, ctx *context.Context, recursive, followSymlinks, getProperties bool, listChan chan string, includeDirectoryStubs bool, incrementEnumerationCounter enumerationCounterFunc, s2sPreserveBlobTags bool, - logLevel pipeline.LogLevel, cpkOptions common.CpkOptions) ResourceTraverser { + logLevel pipeline.LogLevel, cpkOptions common.CpkOptions, syncHashType common.SyncHashType, preservePermissions common.PreservePermissionsOption) ResourceTraverser { var traverserGenerator childTraverserGenerator traverserGenerator = func(relativeChildPath string) (ResourceTraverser, error) { @@ -110,7 +110,7 @@ func newListTraverser(parent common.ResourceString, parentType common.Location, // Construct a traverser that goes through the child traverser, err := InitResourceTraverser(source, parentType, ctx, credential, &followSymlinks, nil, recursive, getProperties, includeDirectoryStubs, common.EPermanentDeleteOption.None(), incrementEnumerationCounter, - nil, s2sPreserveBlobTags, common.ESyncHashType.None(), logLevel, cpkOptions, nil /* errorChannel */) + nil, s2sPreserveBlobTags, syncHashType, preservePermissions, logLevel, cpkOptions, nil /* errorChannel */) if err != nil { return nil, err } diff --git a/cmd/zt_copy_blob_download_test.go b/cmd/zt_copy_blob_download_test.go index 62b488ee4..b13c0850c 100644 --- a/cmd/zt_copy_blob_download_test.go +++ b/cmd/zt_copy_blob_download_test.go @@ -174,7 +174,7 @@ func (s *cmdIntegrationSuite) TestDownloadAccount(c *chk.C) { // Traverse the account ahead of time and determine the relative paths for testing. relPaths := make([]string, 0) // Use a map for easy lookup - blobTraverser := newBlobAccountTraverser(&rawBSU, p, ctx, false, func(common.EntityType) {}, false, common.CpkOptions{}) + blobTraverser := newBlobAccountTraverser(&rawBSU, p, ctx, false, func(common.EntityType) {}, false, common.CpkOptions{}, common.EPreservePermissionsOption.None()) processor := func(object StoredObject) error { // Append the container name to the relative path relPath := "/" + object.ContainerName + "/" + object.relativePath @@ -222,7 +222,7 @@ func (s *cmdIntegrationSuite) TestDownloadAccountWildcard(c *chk.C) { // Traverse the account ahead of time and determine the relative paths for testing. relPaths := make([]string, 0) // Use a map for easy lookup - blobTraverser := newBlobAccountTraverser(&rawBSU, p, ctx, false, func(common.EntityType) {}, false, common.CpkOptions{}) + blobTraverser := newBlobAccountTraverser(&rawBSU, p, ctx, false, func(common.EntityType) {}, false, common.CpkOptions{}, common.EPreservePermissionsOption.None()) processor := func(object StoredObject) error { // Append the container name to the relative path relPath := "/" + object.ContainerName + "/" + object.relativePath diff --git a/cmd/zt_generic_service_traverser_test.go b/cmd/zt_generic_service_traverser_test.go index 33450e609..53033e712 100644 --- a/cmd/zt_generic_service_traverser_test.go +++ b/cmd/zt_generic_service_traverser_test.go @@ -184,7 +184,7 @@ func (s *genericTraverserSuite) TestServiceTraverserWithManyObjects(c *chk.C) { // construct a blob account traverser blobPipeline := azblob.NewPipeline(azblob.NewAnonymousCredential(), azblob.PipelineOptions{}) rawBSU := scenarioHelper{}.getRawBlobServiceURLWithSAS(c) - blobAccountTraverser := newBlobAccountTraverser(&rawBSU, blobPipeline, ctx, false, func(common.EntityType) {}, false, common.CpkOptions{}) + blobAccountTraverser := newBlobAccountTraverser(&rawBSU, blobPipeline, ctx, false, func(common.EntityType) {}, false, common.CpkOptions{}, common.EPreservePermissionsOption.None()) // invoke the blob account traversal with a dummy processor blobDummyProcessor := dummyProcessor{} @@ -369,7 +369,7 @@ func (s *genericTraverserSuite) TestServiceTraverserWithWildcards(c *chk.C) { blobPipeline := azblob.NewPipeline(azblob.NewAnonymousCredential(), azblob.PipelineOptions{}) rawBSU := scenarioHelper{}.getRawBlobServiceURLWithSAS(c) rawBSU.Path = "/objectmatch*" // set the container name to contain a wildcard - blobAccountTraverser := newBlobAccountTraverser(&rawBSU, blobPipeline, ctx, false, func(common.EntityType) {}, false, common.CpkOptions{}) + blobAccountTraverser := newBlobAccountTraverser(&rawBSU, blobPipeline, ctx, false, func(common.EntityType) {}, false, common.CpkOptions{}, common.EPreservePermissionsOption.None()) // invoke the blob account traversal with a dummy processor blobDummyProcessor := dummyProcessor{} diff --git a/cmd/zt_generic_traverser_test.go b/cmd/zt_generic_traverser_test.go index 0c9602dfb..4d9797c9b 100644 --- a/cmd/zt_generic_traverser_test.go +++ b/cmd/zt_generic_traverser_test.go @@ -496,7 +496,7 @@ func (s *genericTraverserSuite) TestTraverserWithSingleObject(c *chk.C) { ctx := context.WithValue(context.TODO(), ste.ServiceAPIVersionOverride, ste.DefaultServiceApiVersion) p := azblob.NewPipeline(azblob.NewAnonymousCredential(), azblob.PipelineOptions{}) rawBlobURLWithSAS := scenarioHelper{}.getRawBlobURLWithSAS(c, containerName, blobList[0]) - blobTraverser := newBlobTraverser(&rawBlobURLWithSAS, p, ctx, false, false, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false) + blobTraverser := newBlobTraverser(&rawBlobURLWithSAS, p, ctx, false, false, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None()) // invoke the blob traversal with a dummy processor blobDummyProcessor := dummyProcessor{} @@ -656,7 +656,7 @@ func (s *genericTraverserSuite) TestTraverserContainerAndLocalDirectory(c *chk.C ctx := context.WithValue(context.TODO(), ste.ServiceAPIVersionOverride, ste.DefaultServiceApiVersion) p := azblob.NewPipeline(azblob.NewAnonymousCredential(), azblob.PipelineOptions{}) rawContainerURLWithSAS := scenarioHelper{}.getRawContainerURLWithSAS(c, containerName) - blobTraverser := newBlobTraverser(&rawContainerURLWithSAS, p, ctx, isRecursiveOn, false, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false) + blobTraverser := newBlobTraverser(&rawContainerURLWithSAS, p, ctx, isRecursiveOn, false, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None()) // invoke the local traversal with a dummy processor blobDummyProcessor := dummyProcessor{} @@ -817,7 +817,7 @@ func (s *genericTraverserSuite) TestTraverserWithVirtualAndLocalDirectory(c *chk ctx := context.WithValue(context.TODO(), ste.ServiceAPIVersionOverride, ste.DefaultServiceApiVersion) p := azblob.NewPipeline(azblob.NewAnonymousCredential(), azblob.PipelineOptions{}) rawVirDirURLWithSAS := scenarioHelper{}.getRawBlobURLWithSAS(c, containerName, virDirName) - blobTraverser := newBlobTraverser(&rawVirDirURLWithSAS, p, ctx, isRecursiveOn, false, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false) + blobTraverser := newBlobTraverser(&rawVirDirURLWithSAS, p, ctx, isRecursiveOn, false, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None()) // invoke the local traversal with a dummy processor blobDummyProcessor := dummyProcessor{} @@ -925,10 +925,10 @@ func (s *genericTraverserSuite) TestSerialAndParallelBlobTraverser(c *chk.C) { ctx := context.WithValue(context.TODO(), ste.ServiceAPIVersionOverride, ste.DefaultServiceApiVersion) p := azblob.NewPipeline(azblob.NewAnonymousCredential(), azblob.PipelineOptions{}) rawVirDirURLWithSAS := scenarioHelper{}.getRawBlobURLWithSAS(c, containerName, virDirName) - parallelBlobTraverser := newBlobTraverser(&rawVirDirURLWithSAS, p, ctx, isRecursiveOn, false, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false) + parallelBlobTraverser := newBlobTraverser(&rawVirDirURLWithSAS, p, ctx, isRecursiveOn, false, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None()) // construct a serial blob traverser - serialBlobTraverser := newBlobTraverser(&rawVirDirURLWithSAS, p, ctx, isRecursiveOn, false, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false) + serialBlobTraverser := newBlobTraverser(&rawVirDirURLWithSAS, p, ctx, isRecursiveOn, false, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None()) serialBlobTraverser.parallelListing = false // invoke the parallel traversal with a dummy processor diff --git a/cmd/zt_traverser_blob_test.go b/cmd/zt_traverser_blob_test.go index cd2eb4879..e77d49cd6 100644 --- a/cmd/zt_traverser_blob_test.go +++ b/cmd/zt_traverser_blob_test.go @@ -48,7 +48,7 @@ func (s *traverserBlobSuite) TestIsSourceDirWithStub(c *chk.C) { // List rawBlobURLWithSAS := scenarioHelper{}.getRawBlobURLWithSAS(c, containerName, dirName) - blobTraverser := newBlobTraverser(&rawBlobURLWithSAS, p, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false) + blobTraverser := newBlobTraverser(&rawBlobURLWithSAS, p, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None()) isDir, err := blobTraverser.IsDirectory(true) c.Assert(isDir, chk.Equals, true) @@ -69,7 +69,7 @@ func (s *traverserBlobSuite) TestIsSourceDirWithNoStub(c *chk.C) { // List rawBlobURLWithSAS := scenarioHelper{}.getRawBlobURLWithSAS(c, containerName, dirName) - blobTraverser := newBlobTraverser(&rawBlobURLWithSAS, p, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false) + blobTraverser := newBlobTraverser(&rawBlobURLWithSAS, p, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None()) isDir, err := blobTraverser.IsDirectory(true) c.Assert(isDir, chk.Equals, true) @@ -92,7 +92,7 @@ func (s *traverserBlobSuite) TestIsSourceFileExists(c *chk.C) { // List rawBlobURLWithSAS := scenarioHelper{}.getRawBlobURLWithSAS(c, containerName, fileName) - blobTraverser := newBlobTraverser(&rawBlobURLWithSAS, p, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false) + blobTraverser := newBlobTraverser(&rawBlobURLWithSAS, p, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None()) isDir, err := blobTraverser.IsDirectory(true) c.Assert(isDir, chk.Equals, false) @@ -113,7 +113,7 @@ func (s *traverserBlobSuite) TestIsSourceFileDoesNotExist(c *chk.C) { // List rawBlobURLWithSAS := scenarioHelper{}.getRawBlobURLWithSAS(c, containerName, fileName) - blobTraverser := newBlobTraverser(&rawBlobURLWithSAS, p, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false) + blobTraverser := newBlobTraverser(&rawBlobURLWithSAS, p, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None()) isDir, err := blobTraverser.IsDirectory(true) c.Assert(isDir, chk.Equals, false) diff --git a/e2etest/declarativeScenario.go b/e2etest/declarativeScenario.go index b911eca02..897b0e7a3 100644 --- a/e2etest/declarativeScenario.go +++ b/e2etest/declarativeScenario.go @@ -27,6 +27,7 @@ import ( "os" "path" "path/filepath" + "strings" "time" "github.com/Azure/azure-storage-azcopy/v10/common" @@ -367,6 +368,15 @@ func (s *scenario) getTransferInfo() (srcRoot string, dstRoot string, expectFold srcRoot = s.state.source.getParam(false, false, "") dstRoot = s.state.dest.getParam(false, false, "") + srcBase := path.Base(srcRoot) + srcRootURL, err := url.Parse(srcRoot) + if err == nil { + snapshotID := srcRootURL.Query().Get("sharesnapshot") + if snapshotID != "" { + srcBase = path.Base(strings.TrimSuffix(srcRoot, "?sharesnapshot="+snapshotID)) + } + } + // do we expect folder transfers expectFolders = (s.fromTo.From().IsFolderAware() && s.fromTo.To().IsFolderAware() && @@ -387,14 +397,14 @@ func (s *scenario) getTransferInfo() (srcRoot string, dstRoot string, expectFold expectRootFolder = false } else if s.fromTo.From().IsLocal() { if tf.objectTarget == "" && tf.destTarget == "" { - addedDirAtDest = filepath.Base(srcRoot) + addedDirAtDest = srcBase } else if tf.destTarget != "" { addedDirAtDest = tf.destTarget } dstRoot = fmt.Sprintf("%s%c%s", dstRoot, os.PathSeparator, addedDirAtDest) } else { if tf.objectTarget == "" && tf.destTarget == "" { - addedDirAtDest = path.Base(srcRoot) + addedDirAtDest = srcBase } else if tf.destTarget != "" { addedDirAtDest = tf.destTarget } From 11e7e3b79985b09a21afe3273269bd1fec6dbf91 Mon Sep 17 00:00:00 2001 From: Adele Reed Date: Tue, 7 Feb 2023 12:33:58 -0800 Subject: [PATCH 4/6] Grab subdir in a different way --- e2etest/declarativeScenario.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/e2etest/declarativeScenario.go b/e2etest/declarativeScenario.go index 897b0e7a3..a47dfa297 100644 --- a/e2etest/declarativeScenario.go +++ b/e2etest/declarativeScenario.go @@ -368,12 +368,12 @@ func (s *scenario) getTransferInfo() (srcRoot string, dstRoot string, expectFold srcRoot = s.state.source.getParam(false, false, "") dstRoot = s.state.dest.getParam(false, false, "") - srcBase := path.Base(srcRoot) + srcBase := filepath.Base(srcRoot) srcRootURL, err := url.Parse(srcRoot) if err == nil { snapshotID := srcRootURL.Query().Get("sharesnapshot") if snapshotID != "" { - srcBase = path.Base(strings.TrimSuffix(srcRoot, "?sharesnapshot="+snapshotID)) + srcBase = filepath.Base(strings.TrimSuffix(srcRoot, "?sharesnapshot="+snapshotID)) } } From a854181f42b563fd3c03291b2125c8509b520bf2 Mon Sep 17 00:00:00 2001 From: Adele Reed Date: Thu, 9 Mar 2023 10:03:54 -0800 Subject: [PATCH 5/6] Fix Build --- cmd/zc_enumerator.go | 4 ++-- cmd/zc_traverser_list.go | 2 -- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/cmd/zc_enumerator.go b/cmd/zc_enumerator.go index 5c6d1b268..aab4dce26 100755 --- a/cmd/zc_enumerator.go +++ b/cmd/zc_enumerator.go @@ -379,7 +379,7 @@ func InitResourceTraverser(resource common.ResourceString, location common.Locat } } - output = newListTraverser(resource, location, credential, ctx, recursive, toFollow, getProperties, + output = newListTraverser(resource, location, credential, ctx, recursive, symlinkHandling, getProperties, listOfFilesChannel, includeDirectoryStubs, incrementEnumerationCounter, s2sPreserveBlobTags, logLevel, cpkOptions, syncHashType, preservePermissions) return output, nil } @@ -407,7 +407,7 @@ func InitResourceTraverser(resource common.ResourceString, location common.Locat }() baseResource := resource.CloneWithValue(cleanLocalPath(basePath)) - output = newListTraverser(baseResource, location, nil, nil, recursive, toFollow, getProperties, + output = newListTraverser(baseResource, location, nil, nil, recursive, symlinkHandling, getProperties, globChan, includeDirectoryStubs, incrementEnumerationCounter, s2sPreserveBlobTags, logLevel, cpkOptions, syncHashType, preservePermissions) } else { if ctx != nil { diff --git a/cmd/zc_traverser_list.go b/cmd/zc_traverser_list.go index 2f67db3e9..9f222d409 100755 --- a/cmd/zc_traverser_list.go +++ b/cmd/zc_traverser_list.go @@ -93,8 +93,6 @@ func newListTraverser(parent common.ResourceString, parentType common.Location, ctx *context.Context, recursive bool, handleSymlinks common.SymlinkHandlingType, getProperties bool, listChan chan string, includeDirectoryStubs bool, incrementEnumerationCounter enumerationCounterFunc, s2sPreserveBlobTags bool, logLevel pipeline.LogLevel, cpkOptions common.CpkOptions, syncHashType common.SyncHashType, preservePermissions common.PreservePermissionsOption) ResourceTraverser { - var traverserGenerator childTraverserGenerator - traverserGenerator := func(relativeChildPath string) (ResourceTraverser, error) { source := parent.Clone() if parentType != common.ELocation.Local() { From f67d7a26a05dec32e124f1146fb6846e49cd6e84 Mon Sep 17 00:00:00 2001 From: Adele Reed Date: Thu, 9 Mar 2023 10:16:44 -0800 Subject: [PATCH 6/6] Lint --- cmd/zc_traverser_blob.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cmd/zc_traverser_blob.go b/cmd/zc_traverser_blob.go index c1961864f..adc19e6c0 100644 --- a/cmd/zc_traverser_blob.go +++ b/cmd/zc_traverser_blob.go @@ -260,6 +260,9 @@ func (t *blobTraverser) Traverse(preprocessor objectMorpher, processor objectPro err := processIfPassedFilters(filters, storedObject, processor) _, err = getProcessingError(err) + if err != nil { + return err + } } // get the container URL so that we can list the blobs