diff --git a/cmd/copyEnumeratorInit.go b/cmd/copyEnumeratorInit.go index 7db16e16b..6ab6c10aa 100755 --- a/cmd/copyEnumeratorInit.go +++ b/cmd/copyEnumeratorInit.go @@ -71,7 +71,7 @@ func (cca *CookedCopyCmdArgs) initEnumerator(jobPartOrder common.CopyJobPartOrde traverser, err = InitResourceTraverser(cca.Source, cca.FromTo.From(), &ctx, &srcCredInfo, cca.SymlinkHandling, cca.ListOfFilesChannel, cca.Recursive, getRemoteProperties, cca.IncludeDirectoryStubs, cca.permanentDeleteOption, func(common.EntityType) {}, cca.ListOfVersionIDs, - cca.S2sPreserveBlobTags, common.ESyncHashType.None(), azcopyLogVerbosity.ToPipelineLogLevel(), cca.CpkOptions, nil /* errorChannel */) + cca.S2sPreserveBlobTags, common.ESyncHashType.None(), cca.preservePermissions, azcopyLogVerbosity.ToPipelineLogLevel(), cca.CpkOptions, nil /* errorChannel */) if err != nil { return nil, err @@ -117,8 +117,8 @@ func (cca *CookedCopyCmdArgs) initEnumerator(jobPartOrder common.CopyJobPartOrde return nil, errors.New("cannot use --as-subdir=false with a service level destination") } - // When copying a container directly to a container, strip the top directory - if srcLevel == ELocationLevel.Container() && dstLevel == ELocationLevel.Container() && cca.FromTo.From().IsRemote() && cca.FromTo.To().IsRemote() { + // When copying a container directly to a container, strip the top directory, unless we're attempting to persist permissions. + if srcLevel == ELocationLevel.Container() && dstLevel == ELocationLevel.Container() && cca.FromTo.From().IsRemote() && cca.FromTo.To().IsRemote() && !cca.preservePermissions.IsTruthy() { cca.StripTopDir = true } @@ -338,7 +338,7 @@ func (cca *CookedCopyCmdArgs) isDestDirectory(dst common.ResourceString, ctx *co rt, err := InitResourceTraverser(dst, cca.FromTo.To(), ctx, &dstCredInfo, common.ESymlinkHandlingType.Skip(), nil, false, false, false, common.EPermanentDeleteOption.None(), - func(common.EntityType) {}, cca.ListOfVersionIDs, false, common.ESyncHashType.None(), pipeline.LogNone, cca.CpkOptions, nil /* errorChannel */) + func(common.EntityType) {}, cca.ListOfVersionIDs, false, common.ESyncHashType.None(), cca.preservePermissions, pipeline.LogNone, cca.CpkOptions, nil /* errorChannel */) if err != nil { return false diff --git a/cmd/copyEnumeratorInit_test.go b/cmd/copyEnumeratorInit_test.go index d99d2e5ca..8a66a088c 100644 --- a/cmd/copyEnumeratorInit_test.go +++ b/cmd/copyEnumeratorInit_test.go @@ -49,7 +49,7 @@ func (ce *copyEnumeratorSuite) TestValidateSourceDirThatExists(c *chk.C) { // List rawBlobURLWithSAS := scenarioHelper{}.getRawBlobURLWithSAS(c, containerName, dirName) - blobTraverser := newBlobTraverser(&rawBlobURLWithSAS, p, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false) + blobTraverser := newBlobTraverser(&rawBlobURLWithSAS, p, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None()) // dir but recursive flag not set - fail cca := CookedCopyCmdArgs{StripTopDir: false, Recursive: false} @@ -78,7 +78,7 @@ func (ce *copyEnumeratorSuite) TestValidateSourceDirDoesNotExist(c *chk.C) { // List rawBlobURLWithSAS := scenarioHelper{}.getRawBlobURLWithSAS(c, containerName, dirName) - blobTraverser := newBlobTraverser(&rawBlobURLWithSAS, p, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false) + blobTraverser := newBlobTraverser(&rawBlobURLWithSAS, p, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None()) // dir but recursive flag not set - fail cca := CookedCopyCmdArgs{StripTopDir: false, Recursive: false} @@ -108,7 +108,7 @@ func (ce *copyEnumeratorSuite) TestValidateSourceFileExists(c *chk.C) { // List rawBlobURLWithSAS := scenarioHelper{}.getRawBlobURLWithSAS(c, containerName, fileName) - blobTraverser := newBlobTraverser(&rawBlobURLWithSAS, p, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false) + blobTraverser := newBlobTraverser(&rawBlobURLWithSAS, p, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None()) cca := CookedCopyCmdArgs{StripTopDir: false, Recursive: false} err := cca.validateSourceDir(blobTraverser) @@ -131,7 +131,7 @@ func (ce *copyEnumeratorSuite) TestValidateSourceFileDoesNotExist(c *chk.C) { // List rawBlobURLWithSAS := scenarioHelper{}.getRawBlobURLWithSAS(c, containerName, fileName) - blobTraverser := newBlobTraverser(&rawBlobURLWithSAS, p, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false) + blobTraverser := newBlobTraverser(&rawBlobURLWithSAS, p, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None()) cca := CookedCopyCmdArgs{StripTopDir: false, Recursive: false} err := cca.validateSourceDir(blobTraverser) @@ -154,7 +154,7 @@ func (ce *copyEnumeratorSuite) TestValidateSourceWithWildCard(c *chk.C) { // List rawBlobURLWithSAS := scenarioHelper{}.getRawBlobURLWithSAS(c, containerName, dirName) - blobTraverser := newBlobTraverser(&rawBlobURLWithSAS, p, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false) + blobTraverser := newBlobTraverser(&rawBlobURLWithSAS, p, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None()) // dir but recursive flag not set - fail cca := CookedCopyCmdArgs{StripTopDir: true, Recursive: false} diff --git a/cmd/list.go b/cmd/list.go index 851953321..e7d08bbf3 100755 --- a/cmd/list.go +++ b/cmd/list.go @@ -224,7 +224,7 @@ func (cooked cookedListCmdArgs) HandleListContainerCommand() (err error) { traverser, err := InitResourceTraverser(source, cooked.location, &ctx, &credentialInfo, common.ESymlinkHandlingType.Skip(), nil, true, false, false, common.EPermanentDeleteOption.None(), func(common.EntityType) {}, - nil, false, common.ESyncHashType.None(), pipeline.LogNone, common.CpkOptions{}, nil /* errorChannel */) + nil, false, common.ESyncHashType.None(), common.EPreservePermissionsOption.None(), pipeline.LogNone, common.CpkOptions{}, nil /* errorChannel */) if err != nil { return fmt.Errorf("failed to initialize traverser: %s", err.Error()) diff --git a/cmd/removeEnumerator.go b/cmd/removeEnumerator.go index 063efff51..fabdfd307 100755 --- a/cmd/removeEnumerator.go +++ b/cmd/removeEnumerator.go @@ -51,7 +51,7 @@ func newRemoveEnumerator(cca *CookedCopyCmdArgs) (enumerator *CopyEnumerator, er sourceTraverser, err = InitResourceTraverser(cca.Source, cca.FromTo.From(), &ctx, &cca.credentialInfo, common.ESymlinkHandlingType.Skip(), cca.ListOfFilesChannel, cca.Recursive, false, cca.IncludeDirectoryStubs, cca.permanentDeleteOption, func(common.EntityType) {}, cca.ListOfVersionIDs, false, - common.ESyncHashType.None(), azcopyLogVerbosity.ToPipelineLogLevel(), cca.CpkOptions, nil /* errorChannel */) + common.ESyncHashType.None(), common.EPreservePermissionsOption.None(), azcopyLogVerbosity.ToPipelineLogLevel(), cca.CpkOptions, nil /* errorChannel */) // report failure to create traverser if err != nil { diff --git a/cmd/setPropertiesEnumerator.go b/cmd/setPropertiesEnumerator.go index 8fa96a337..98a209955 100755 --- a/cmd/setPropertiesEnumerator.go +++ b/cmd/setPropertiesEnumerator.go @@ -51,7 +51,7 @@ func setPropertiesEnumerator(cca *CookedCopyCmdArgs) (enumerator *CopyEnumerator common.ESymlinkHandlingType.Preserve(), // preserve because we want to index all blobs, including symlink blobs cca.ListOfFilesChannel, cca.Recursive, false, cca.IncludeDirectoryStubs, cca.permanentDeleteOption, func(common.EntityType) {}, cca.ListOfVersionIDs, false, - common.ESyncHashType.None(), azcopyLogVerbosity.ToPipelineLogLevel(), cca.CpkOptions, nil /* errorChannel */) + common.ESyncHashType.None(), common.EPreservePermissionsOption.None(), azcopyLogVerbosity.ToPipelineLogLevel(), cca.CpkOptions, nil /* errorChannel */) // report failure to create traverser if err != nil { diff --git a/cmd/syncEnumerator.go b/cmd/syncEnumerator.go index 227793c66..5ac900abf 100644 --- a/cmd/syncEnumerator.go +++ b/cmd/syncEnumerator.go @@ -65,7 +65,7 @@ func (cca *cookedSyncCmdArgs) initEnumerator(ctx context.Context) (enumerator *s if entityType == common.EEntityType.File() { atomic.AddUint64(&cca.atomicSourceFilesScanned, 1) } - }, nil, cca.s2sPreserveBlobTags, cca.compareHash, azcopyLogVerbosity.ToPipelineLogLevel(), cca.cpkOptions, nil /* errorChannel */) + }, nil, cca.s2sPreserveBlobTags, cca.compareHash, cca.preservePermissions, azcopyLogVerbosity.ToPipelineLogLevel(), cca.cpkOptions, nil /* errorChannel */) if err != nil { return nil, err @@ -86,7 +86,7 @@ func (cca *cookedSyncCmdArgs) initEnumerator(ctx context.Context) (enumerator *s if entityType == common.EEntityType.File() { atomic.AddUint64(&cca.atomicDestinationFilesScanned, 1) } - }, nil, cca.s2sPreserveBlobTags, cca.compareHash, azcopyLogVerbosity.ToPipelineLogLevel(), cca.cpkOptions, nil /* errorChannel */) + }, nil, cca.s2sPreserveBlobTags, cca.compareHash, cca.preservePermissions, azcopyLogVerbosity.ToPipelineLogLevel(), cca.cpkOptions, nil /* errorChannel */) if err != nil { return nil, err } diff --git a/cmd/zc_enumerator.go b/cmd/zc_enumerator.go old mode 100644 new mode 100755 index a83b91357..aab4dce26 --- a/cmd/zc_enumerator.go +++ b/cmd/zc_enumerator.go @@ -332,7 +332,7 @@ type enumerationCounterFunc func(entityType common.EntityType) func InitResourceTraverser(resource common.ResourceString, location common.Location, ctx *context.Context, credential *common.CredentialInfo, symlinkHandling common.SymlinkHandlingType, listOfFilesChannel chan string, recursive, getProperties, includeDirectoryStubs bool, permanentDeleteOption common.PermanentDeleteOption, incrementEnumerationCounter enumerationCounterFunc, listOfVersionIds chan string, - s2sPreserveBlobTags bool, syncHashType common.SyncHashType, logLevel pipeline.LogLevel, cpkOptions common.CpkOptions, errorChannel chan ErrorFileInfo) (ResourceTraverser, error) { + s2sPreserveBlobTags bool, syncHashType common.SyncHashType, preservePermissions common.PreservePermissionsOption, logLevel pipeline.LogLevel, cpkOptions common.CpkOptions, errorChannel chan ErrorFileInfo) (ResourceTraverser, error) { var output ResourceTraverser var p *pipeline.Pipeline @@ -380,7 +380,7 @@ func InitResourceTraverser(resource common.ResourceString, location common.Locat } output = newListTraverser(resource, location, credential, ctx, recursive, symlinkHandling, getProperties, - listOfFilesChannel, includeDirectoryStubs, incrementEnumerationCounter, s2sPreserveBlobTags, logLevel, cpkOptions) + listOfFilesChannel, includeDirectoryStubs, incrementEnumerationCounter, s2sPreserveBlobTags, logLevel, cpkOptions, syncHashType, preservePermissions) return output, nil } @@ -408,7 +408,7 @@ func InitResourceTraverser(resource common.ResourceString, location common.Locat baseResource := resource.CloneWithValue(cleanLocalPath(basePath)) output = newListTraverser(baseResource, location, nil, nil, recursive, symlinkHandling, getProperties, - globChan, includeDirectoryStubs, incrementEnumerationCounter, s2sPreserveBlobTags, logLevel, cpkOptions) + globChan, includeDirectoryStubs, incrementEnumerationCounter, s2sPreserveBlobTags, logLevel, cpkOptions, syncHashType, preservePermissions) } else { if ctx != nil { output = newLocalTraverser(*ctx, resource.ValueLocal(), recursive, symlinkHandling, syncHashType, incrementEnumerationCounter, errorChannel) @@ -443,11 +443,11 @@ func InitResourceTraverser(resource common.ResourceString, location common.Locat return nil, errors.New(accountTraversalInherentlyRecursiveError) } - output = newBlobAccountTraverser(resourceURL, *p, *ctx, includeDirectoryStubs, incrementEnumerationCounter, s2sPreserveBlobTags, cpkOptions) + output = newBlobAccountTraverser(resourceURL, *p, *ctx, includeDirectoryStubs, incrementEnumerationCounter, s2sPreserveBlobTags, cpkOptions, preservePermissions) } else if listOfVersionIds != nil { output = newBlobVersionsTraverser(resourceURL, *p, *ctx, recursive, includeDirectoryStubs, incrementEnumerationCounter, listOfVersionIds, cpkOptions) } else { - output = newBlobTraverser(resourceURL, *p, *ctx, recursive, includeDirectoryStubs, incrementEnumerationCounter, s2sPreserveBlobTags, cpkOptions, includeDeleted, includeSnapshot, includeVersion) + output = newBlobTraverser(resourceURL, *p, *ctx, recursive, includeDirectoryStubs, incrementEnumerationCounter, s2sPreserveBlobTags, cpkOptions, includeDeleted, includeSnapshot, includeVersion, preservePermissions) } case common.ELocation.File(): resourceURL, err := resource.FullURL() diff --git a/cmd/zc_traverser_blob.go b/cmd/zc_traverser_blob.go index 3e4cbc361..adc19e6c0 100644 --- a/cmd/zc_traverser_blob.go +++ b/cmd/zc_traverser_blob.go @@ -25,6 +25,7 @@ import ( "fmt" "net/url" "strings" + "time" "github.com/Azure/azure-storage-azcopy/v10/common/parallel" @@ -56,6 +57,8 @@ type blobTraverser struct { cpkOptions common.CpkOptions + preservePermissions common.PreservePermissionsOption + includeDeleted bool includeSnapshot bool @@ -222,7 +225,7 @@ func (t *blobTraverser) Traverse(preprocessor objectMorpher, processor objectPro } } if t.incrementEnumerationCounter != nil { - t.incrementEnumerationCounter(common.EEntityType.File()) + t.incrementEnumerationCounter(storedObject.entityType) } err := processIfPassedFilters(filters, storedObject, processor) @@ -232,6 +235,34 @@ func (t *blobTraverser) Traverse(preprocessor objectMorpher, processor objectPro if !t.includeDeleted && (isBlob || err != nil) { return err } + } else if blobUrlParts.BlobName == "" && t.preservePermissions.IsTruthy() { + // if the root is a container and we're copying "folders", we should persist the ACLs there too. + if azcopyScanningLogger != nil { + azcopyScanningLogger.Log(pipeline.LogDebug, "Detected the root as a container.") + } + + storedObject := newStoredObject( + preprocessor, + "", + "", + common.EEntityType.Folder(), + time.Now(), + 0, + noContentProps, + noBlobProps, + common.Metadata{}, + blobUrlParts.ContainerName, + ) + + if t.incrementEnumerationCounter != nil { + t.incrementEnumerationCounter(common.EEntityType.Folder()) + } + + err := processIfPassedFilters(filters, storedObject, processor) + _, err = getProcessingError(err) + if err != nil { + return err + } } // get the container URL so that we can list the blobs @@ -487,7 +518,7 @@ func (t *blobTraverser) serialList(containerURL azblob.ContainerURL, containerNa return nil } -func newBlobTraverser(rawURL *url.URL, p pipeline.Pipeline, ctx context.Context, recursive, includeDirectoryStubs bool, incrementEnumerationCounter enumerationCounterFunc, s2sPreserveSourceTags bool, cpkOptions common.CpkOptions, includeDeleted, includeSnapshot, includeVersion bool) (t *blobTraverser) { +func newBlobTraverser(rawURL *url.URL, p pipeline.Pipeline, ctx context.Context, recursive, includeDirectoryStubs bool, incrementEnumerationCounter enumerationCounterFunc, s2sPreserveSourceTags bool, cpkOptions common.CpkOptions, includeDeleted, includeSnapshot, includeVersion bool, preservePermissions common.PreservePermissionsOption) (t *blobTraverser) { t = &blobTraverser{ rawURL: rawURL, p: p, @@ -501,6 +532,7 @@ func newBlobTraverser(rawURL *url.URL, p pipeline.Pipeline, ctx context.Context, includeDeleted: includeDeleted, includeSnapshot: includeSnapshot, includeVersion: includeVersion, + preservePermissions: preservePermissions, } disableHierarchicalScanning := strings.ToLower(glcm.GetEnvironmentVariable(common.EEnvironmentVariable.DisableHierarchicalScanning())) diff --git a/cmd/zc_traverser_blob_account.go b/cmd/zc_traverser_blob_account.go index 6c946e01c..b5c3f3980 100644 --- a/cmd/zc_traverser_blob_account.go +++ b/cmd/zc_traverser_blob_account.go @@ -45,6 +45,7 @@ type blobAccountTraverser struct { s2sPreserveSourceTags bool cpkOptions common.CpkOptions + preservePermissions common.PreservePermissionsOption } func (t *blobAccountTraverser) IsDirectory(_ bool) (bool, error) { @@ -100,7 +101,7 @@ func (t *blobAccountTraverser) Traverse(preprocessor objectMorpher, processor ob for _, v := range cList { containerURL := t.accountURL.NewContainerURL(v).URL() - containerTraverser := newBlobTraverser(&containerURL, t.p, t.ctx, true, t.includeDirectoryStubs, t.incrementEnumerationCounter, t.s2sPreserveSourceTags, t.cpkOptions, false, false, false) + containerTraverser := newBlobTraverser(&containerURL, t.p, t.ctx, true, t.includeDirectoryStubs, t.incrementEnumerationCounter, t.s2sPreserveSourceTags, t.cpkOptions, false, false, false, t.preservePermissions) preprocessorForThisChild := preprocessor.FollowedBy(newContainerDecorator(v)) @@ -115,7 +116,7 @@ func (t *blobAccountTraverser) Traverse(preprocessor objectMorpher, processor ob return nil } -func newBlobAccountTraverser(rawURL *url.URL, p pipeline.Pipeline, ctx context.Context, includeDirectoryStubs bool, incrementEnumerationCounter enumerationCounterFunc, s2sPreserveSourceTags bool, cpkOptions common.CpkOptions) (t *blobAccountTraverser) { +func newBlobAccountTraverser(rawURL *url.URL, p pipeline.Pipeline, ctx context.Context, includeDirectoryStubs bool, incrementEnumerationCounter enumerationCounterFunc, s2sPreserveSourceTags bool, cpkOptions common.CpkOptions, preservePermissions common.PreservePermissionsOption) (t *blobAccountTraverser) { bURLParts := azblob.NewBlobURLParts(*rawURL) cPattern := bURLParts.ContainerName @@ -133,6 +134,7 @@ func newBlobAccountTraverser(rawURL *url.URL, p pipeline.Pipeline, ctx context.C includeDirectoryStubs: includeDirectoryStubs, s2sPreserveSourceTags: s2sPreserveSourceTags, cpkOptions: cpkOptions, + preservePermissions: preservePermissions, } return diff --git a/cmd/zc_traverser_list.go b/cmd/zc_traverser_list.go index d4a4d420a..9f222d409 100755 --- a/cmd/zc_traverser_list.go +++ b/cmd/zc_traverser_list.go @@ -92,8 +92,7 @@ func (l *listTraverser) Traverse(preprocessor objectMorpher, processor objectPro func newListTraverser(parent common.ResourceString, parentType common.Location, credential *common.CredentialInfo, ctx *context.Context, recursive bool, handleSymlinks common.SymlinkHandlingType, getProperties bool, listChan chan string, includeDirectoryStubs bool, incrementEnumerationCounter enumerationCounterFunc, s2sPreserveBlobTags bool, - logLevel pipeline.LogLevel, cpkOptions common.CpkOptions) ResourceTraverser { - + logLevel pipeline.LogLevel, cpkOptions common.CpkOptions, syncHashType common.SyncHashType, preservePermissions common.PreservePermissionsOption) ResourceTraverser { traverserGenerator := func(relativeChildPath string) (ResourceTraverser, error) { source := parent.Clone() if parentType != common.ELocation.Local() { @@ -109,7 +108,7 @@ func newListTraverser(parent common.ResourceString, parentType common.Location, // Construct a traverser that goes through the child traverser, err := InitResourceTraverser(source, parentType, ctx, credential, handleSymlinks, nil, recursive, getProperties, includeDirectoryStubs, common.EPermanentDeleteOption.None(), incrementEnumerationCounter, - nil, s2sPreserveBlobTags, common.ESyncHashType.None(), logLevel, cpkOptions, nil /* errorChannel */) + nil, s2sPreserveBlobTags, syncHashType, preservePermissions, logLevel, cpkOptions, nil /* errorChannel */) if err != nil { return nil, err } diff --git a/cmd/zt_copy_blob_download_test.go b/cmd/zt_copy_blob_download_test.go index 62b488ee4..b13c0850c 100644 --- a/cmd/zt_copy_blob_download_test.go +++ b/cmd/zt_copy_blob_download_test.go @@ -174,7 +174,7 @@ func (s *cmdIntegrationSuite) TestDownloadAccount(c *chk.C) { // Traverse the account ahead of time and determine the relative paths for testing. relPaths := make([]string, 0) // Use a map for easy lookup - blobTraverser := newBlobAccountTraverser(&rawBSU, p, ctx, false, func(common.EntityType) {}, false, common.CpkOptions{}) + blobTraverser := newBlobAccountTraverser(&rawBSU, p, ctx, false, func(common.EntityType) {}, false, common.CpkOptions{}, common.EPreservePermissionsOption.None()) processor := func(object StoredObject) error { // Append the container name to the relative path relPath := "/" + object.ContainerName + "/" + object.relativePath @@ -222,7 +222,7 @@ func (s *cmdIntegrationSuite) TestDownloadAccountWildcard(c *chk.C) { // Traverse the account ahead of time and determine the relative paths for testing. relPaths := make([]string, 0) // Use a map for easy lookup - blobTraverser := newBlobAccountTraverser(&rawBSU, p, ctx, false, func(common.EntityType) {}, false, common.CpkOptions{}) + blobTraverser := newBlobAccountTraverser(&rawBSU, p, ctx, false, func(common.EntityType) {}, false, common.CpkOptions{}, common.EPreservePermissionsOption.None()) processor := func(object StoredObject) error { // Append the container name to the relative path relPath := "/" + object.ContainerName + "/" + object.relativePath diff --git a/cmd/zt_generic_service_traverser_test.go b/cmd/zt_generic_service_traverser_test.go index 05f399a5e..3641a8142 100644 --- a/cmd/zt_generic_service_traverser_test.go +++ b/cmd/zt_generic_service_traverser_test.go @@ -184,7 +184,7 @@ func (s *genericTraverserSuite) TestServiceTraverserWithManyObjects(c *chk.C) { // construct a blob account traverser blobPipeline := azblob.NewPipeline(azblob.NewAnonymousCredential(), azblob.PipelineOptions{}) rawBSU := scenarioHelper{}.getRawBlobServiceURLWithSAS(c) - blobAccountTraverser := newBlobAccountTraverser(&rawBSU, blobPipeline, ctx, false, func(common.EntityType) {}, false, common.CpkOptions{}) + blobAccountTraverser := newBlobAccountTraverser(&rawBSU, blobPipeline, ctx, false, func(common.EntityType) {}, false, common.CpkOptions{}, common.EPreservePermissionsOption.None()) // invoke the blob account traversal with a dummy processor blobDummyProcessor := dummyProcessor{} @@ -369,7 +369,7 @@ func (s *genericTraverserSuite) TestServiceTraverserWithWildcards(c *chk.C) { blobPipeline := azblob.NewPipeline(azblob.NewAnonymousCredential(), azblob.PipelineOptions{}) rawBSU := scenarioHelper{}.getRawBlobServiceURLWithSAS(c) rawBSU.Path = "/objectmatch*" // set the container name to contain a wildcard - blobAccountTraverser := newBlobAccountTraverser(&rawBSU, blobPipeline, ctx, false, func(common.EntityType) {}, false, common.CpkOptions{}) + blobAccountTraverser := newBlobAccountTraverser(&rawBSU, blobPipeline, ctx, false, func(common.EntityType) {}, false, common.CpkOptions{}, common.EPreservePermissionsOption.None()) // invoke the blob account traversal with a dummy processor blobDummyProcessor := dummyProcessor{} diff --git a/cmd/zt_generic_traverser_test.go b/cmd/zt_generic_traverser_test.go index a75a385d9..3db10c8c4 100644 --- a/cmd/zt_generic_traverser_test.go +++ b/cmd/zt_generic_traverser_test.go @@ -495,7 +495,7 @@ func (s *genericTraverserSuite) TestTraverserWithSingleObject(c *chk.C) { ctx := context.WithValue(context.TODO(), ste.ServiceAPIVersionOverride, ste.DefaultServiceApiVersion) p := azblob.NewPipeline(azblob.NewAnonymousCredential(), azblob.PipelineOptions{}) rawBlobURLWithSAS := scenarioHelper{}.getRawBlobURLWithSAS(c, containerName, blobList[0]) - blobTraverser := newBlobTraverser(&rawBlobURLWithSAS, p, ctx, false, false, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false) + blobTraverser := newBlobTraverser(&rawBlobURLWithSAS, p, ctx, false, false, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None()) // invoke the blob traversal with a dummy processor blobDummyProcessor := dummyProcessor{} @@ -655,7 +655,7 @@ func (s *genericTraverserSuite) TestTraverserContainerAndLocalDirectory(c *chk.C ctx := context.WithValue(context.TODO(), ste.ServiceAPIVersionOverride, ste.DefaultServiceApiVersion) p := azblob.NewPipeline(azblob.NewAnonymousCredential(), azblob.PipelineOptions{}) rawContainerURLWithSAS := scenarioHelper{}.getRawContainerURLWithSAS(c, containerName) - blobTraverser := newBlobTraverser(&rawContainerURLWithSAS, p, ctx, isRecursiveOn, false, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false) + blobTraverser := newBlobTraverser(&rawContainerURLWithSAS, p, ctx, isRecursiveOn, false, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None()) // invoke the local traversal with a dummy processor blobDummyProcessor := dummyProcessor{} @@ -816,7 +816,7 @@ func (s *genericTraverserSuite) TestTraverserWithVirtualAndLocalDirectory(c *chk ctx := context.WithValue(context.TODO(), ste.ServiceAPIVersionOverride, ste.DefaultServiceApiVersion) p := azblob.NewPipeline(azblob.NewAnonymousCredential(), azblob.PipelineOptions{}) rawVirDirURLWithSAS := scenarioHelper{}.getRawBlobURLWithSAS(c, containerName, virDirName) - blobTraverser := newBlobTraverser(&rawVirDirURLWithSAS, p, ctx, isRecursiveOn, false, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false) + blobTraverser := newBlobTraverser(&rawVirDirURLWithSAS, p, ctx, isRecursiveOn, false, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None()) // invoke the local traversal with a dummy processor blobDummyProcessor := dummyProcessor{} @@ -924,10 +924,10 @@ func (s *genericTraverserSuite) TestSerialAndParallelBlobTraverser(c *chk.C) { ctx := context.WithValue(context.TODO(), ste.ServiceAPIVersionOverride, ste.DefaultServiceApiVersion) p := azblob.NewPipeline(azblob.NewAnonymousCredential(), azblob.PipelineOptions{}) rawVirDirURLWithSAS := scenarioHelper{}.getRawBlobURLWithSAS(c, containerName, virDirName) - parallelBlobTraverser := newBlobTraverser(&rawVirDirURLWithSAS, p, ctx, isRecursiveOn, false, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false) + parallelBlobTraverser := newBlobTraverser(&rawVirDirURLWithSAS, p, ctx, isRecursiveOn, false, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None()) // construct a serial blob traverser - serialBlobTraverser := newBlobTraverser(&rawVirDirURLWithSAS, p, ctx, isRecursiveOn, false, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false) + serialBlobTraverser := newBlobTraverser(&rawVirDirURLWithSAS, p, ctx, isRecursiveOn, false, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None()) serialBlobTraverser.parallelListing = false // invoke the parallel traversal with a dummy processor diff --git a/cmd/zt_traverser_blob_test.go b/cmd/zt_traverser_blob_test.go index cd2eb4879..e77d49cd6 100644 --- a/cmd/zt_traverser_blob_test.go +++ b/cmd/zt_traverser_blob_test.go @@ -48,7 +48,7 @@ func (s *traverserBlobSuite) TestIsSourceDirWithStub(c *chk.C) { // List rawBlobURLWithSAS := scenarioHelper{}.getRawBlobURLWithSAS(c, containerName, dirName) - blobTraverser := newBlobTraverser(&rawBlobURLWithSAS, p, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false) + blobTraverser := newBlobTraverser(&rawBlobURLWithSAS, p, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None()) isDir, err := blobTraverser.IsDirectory(true) c.Assert(isDir, chk.Equals, true) @@ -69,7 +69,7 @@ func (s *traverserBlobSuite) TestIsSourceDirWithNoStub(c *chk.C) { // List rawBlobURLWithSAS := scenarioHelper{}.getRawBlobURLWithSAS(c, containerName, dirName) - blobTraverser := newBlobTraverser(&rawBlobURLWithSAS, p, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false) + blobTraverser := newBlobTraverser(&rawBlobURLWithSAS, p, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None()) isDir, err := blobTraverser.IsDirectory(true) c.Assert(isDir, chk.Equals, true) @@ -92,7 +92,7 @@ func (s *traverserBlobSuite) TestIsSourceFileExists(c *chk.C) { // List rawBlobURLWithSAS := scenarioHelper{}.getRawBlobURLWithSAS(c, containerName, fileName) - blobTraverser := newBlobTraverser(&rawBlobURLWithSAS, p, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false) + blobTraverser := newBlobTraverser(&rawBlobURLWithSAS, p, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None()) isDir, err := blobTraverser.IsDirectory(true) c.Assert(isDir, chk.Equals, false) @@ -113,7 +113,7 @@ func (s *traverserBlobSuite) TestIsSourceFileDoesNotExist(c *chk.C) { // List rawBlobURLWithSAS := scenarioHelper{}.getRawBlobURLWithSAS(c, containerName, fileName) - blobTraverser := newBlobTraverser(&rawBlobURLWithSAS, p, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false) + blobTraverser := newBlobTraverser(&rawBlobURLWithSAS, p, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None()) isDir, err := blobTraverser.IsDirectory(true) c.Assert(isDir, chk.Equals, false) diff --git a/e2etest/declarativeHelpers.go b/e2etest/declarativeHelpers.go index d1443c83c..f3a825430 100644 --- a/e2etest/declarativeHelpers.go +++ b/e2etest/declarativeHelpers.go @@ -115,7 +115,7 @@ func (a *testingAsserter) AssertNoErr(err error, comment ...string) { a.t.Helper() // exclude this method from the logged callstack redactedErr := sanitizer.SanitizeLogMessage(err.Error()) a.t.Logf("Error %s%s", redactedErr, a.formatComments(comment)) - a.t.FailNow() + a.t.Fail() } } diff --git a/e2etest/declarativeScenario.go b/e2etest/declarativeScenario.go index dadbd3634..0fe5950c1 100644 --- a/e2etest/declarativeScenario.go +++ b/e2etest/declarativeScenario.go @@ -27,6 +27,7 @@ import ( "os" "path" "path/filepath" + "strings" "time" "github.com/Azure/azure-storage-azcopy/v10/common" @@ -367,6 +368,15 @@ func (s *scenario) getTransferInfo() (srcRoot string, dstRoot string, expectFold srcRoot = s.state.source.getParam(false, false, "") dstRoot = s.state.dest.getParam(false, false, "") + srcBase := filepath.Base(srcRoot) + srcRootURL, err := url.Parse(srcRoot) + if err == nil { + snapshotID := srcRootURL.Query().Get("sharesnapshot") + if snapshotID != "" { + srcBase = filepath.Base(strings.TrimSuffix(srcRoot, "?sharesnapshot="+snapshotID)) + } + } + // do we expect folder transfers expectFolders = (s.fromTo.From().IsFolderAware() && s.fromTo.To().IsFolderAware() && @@ -377,7 +387,7 @@ func (s *scenario) getTransferInfo() (srcRoot string, dstRoot string, expectFold // compute dest, taking into account our stripToDir rules addedDirAtDest = "" - areBothContainerLike := s.state.source.isContainerLike() && s.state.dest.isContainerLike() + areBothContainerLike := s.state.source.isContainerLike() && s.state.dest.isContainerLike() && !s.p.preserveSMBPermissions // There are no permission-compatible sources and destinations that do not feature support for root folder perms anymore* tf := s.GetTestFiles() if s.stripTopDir || s.operation == eOperation.Sync() || areBothContainerLike { @@ -388,14 +398,14 @@ func (s *scenario) getTransferInfo() (srcRoot string, dstRoot string, expectFold expectRootFolder = false } else if s.fromTo.From().IsLocal() { if tf.objectTarget == "" && tf.destTarget == "" { - addedDirAtDest = filepath.Base(srcRoot) + addedDirAtDest = srcBase } else if tf.destTarget != "" { addedDirAtDest = tf.destTarget } dstRoot = fmt.Sprintf("%s%c%s", dstRoot, os.PathSeparator, addedDirAtDest) } else { if tf.objectTarget == "" && tf.destTarget == "" { - addedDirAtDest = path.Base(srcRoot) + addedDirAtDest = srcBase } else if tf.destTarget != "" { addedDirAtDest = tf.destTarget } diff --git a/e2etest/zt_preserve_properties_test.go b/e2etest/zt_preserve_properties_test.go index d94a9514e..a37c66014 100644 --- a/e2etest/zt_preserve_properties_test.go +++ b/e2etest/zt_preserve_properties_test.go @@ -62,7 +62,7 @@ func TestProperties_HNSACLs(t *testing.T) { }, nil, testFiles{ defaultSize: "1K", shouldTransfer: []interface{}{ - folder(""), + folder("", with{adlsPermissionsACL: "user::rwx,group::rw-,other::r--"}), f("filea", with{adlsPermissionsACL: "user::rwx,group::rwx,other::r--"}), folder("a", with{adlsPermissionsACL: "user::rwx,group::rwx,other::-w-"}), f("a/fileb", with{adlsPermissionsACL: "user::rwx,group::rwx,other::--x"}), diff --git a/ste/sender-blobFolders.go b/ste/sender-blobFolders.go index 756c5fe43..2a2f53fba 100644 --- a/ste/sender-blobFolders.go +++ b/ste/sender-blobFolders.go @@ -76,22 +76,25 @@ func (b *blobFolderSender) setDatalakeACLs() { func (b *blobFolderSender) overwriteDFSProperties() (string, error) { b.jptm.Log(pipeline.LogWarning, "It is impossible to completely overwrite a folder with existing content under it on a hierarchical namespace storage account. A best-effort attempt will be made, but if CPK does not match the transfer will fail.") - b.metadataToApply["hdi_isfolder"] = "true" // Set folder metadata flag err := b.getExtraProperties() if err != nil { return "Get Extra Properties", fmt.Errorf("when getting additional folder properties: %w", err) } + // do not set folder flag as it's invalid to modify a folder with + delete(b.metadataToApply, "hdi_isfolder") + // SetMetadata can set CPK if it wasn't specified prior. This is not a "full" overwrite, but a best-effort overwrite. _, err = b.destination.SetMetadata(b.jptm.Context(), b.metadataToApply, azblob.BlobAccessConditions{}, b.cpkToApply) if err != nil { return "Set Metadata", fmt.Errorf("A best-effort overwrite was attempted; CPK errors cannot be handled when the blob cannot be deleted.\n%w", err) } - _, err = b.destination.SetTags(b.jptm.Context(), nil, nil, nil, b.blobTagsToApply) - if err != nil { - return "Set Blob Tags", err - } + // blob API not yet supported for HNS account error; re-enable later. + //_, err = b.destination.SetTags(b.jptm.Context(), nil, nil, nil, b.blobTagsToApply) + //if err != nil { + // return "Set Blob Tags", err + //} _, err = b.destination.SetHTTPHeaders(b.jptm.Context(), b.headersToAppply, azblob.BlobAccessConditions{}) if err != nil { return "Set HTTP Headers", err @@ -105,9 +108,36 @@ func (b *blobFolderSender) overwriteDFSProperties() (string, error) { return "", nil } +func (b *blobFolderSender) SetContainerACL() error { + bURLParts := azblob.NewBlobURLParts(b.destination.URL()) + bURLParts.BlobName = "/" // Container-level ACLs NEED a / + bURLParts.Host = strings.ReplaceAll(bURLParts.Host, ".blob", ".dfs") + // todo: jank, and violates the principle of interfaces + fileURL := azbfs.NewFileSystemURL(bURLParts.URL(), b.jptm.(*jobPartTransferMgr).jobPartMgr.(*jobPartMgr).secondaryPipeline) + + // We know for a fact our source is a "blob". + acl, err := b.sip.(*blobSourceInfoProvider).AccessControl() + if err != nil { + b.jptm.FailActiveSend("Grabbing source ACLs", err) + return folderPropertiesSetInCreation{} // standard completion will detect failure + } + acl.Permissions = "" // Since we're sending the full ACL, Permissions is irrelevant. + _, err = fileURL.SetAccessControl(b.jptm.Context(), acl) + if err != nil { + b.jptm.FailActiveSend("Putting ACLs", err) + return folderPropertiesSetInCreation{} // standard completion will detect failure + } + + return folderPropertiesSetInCreation{} // standard completion will handle the rest +} + func (b *blobFolderSender) EnsureFolderExists() error { t := b.jptm.GetFolderCreationTracker() + if azblob.NewBlobURLParts(b.destination.URL()).BlobName == "" { + return b.SetContainerACL() // Can't do much with a container, but it is here. + } + _, err := b.destination.GetProperties(b.jptm.Context(), azblob.BlobAccessConditions{}, b.cpkToApply) if err != nil { if stgErr, ok := err.(azblob.StorageError); !(ok && stgErr.ServiceCode() == azblob.ServiceCodeBlobNotFound) { diff --git a/ste/sourceInfoProvider-Blob.go b/ste/sourceInfoProvider-Blob.go index 7f3aa9644..f0de67003 100644 --- a/ste/sourceInfoProvider-Blob.go +++ b/ste/sourceInfoProvider-Blob.go @@ -112,10 +112,14 @@ func (p *blobSourceInfoProvider) AccessControl() (azbfs.BlobFSAccessControl, err bURLParts := azblob.NewBlobURLParts(*presignedURL) bURLParts.Host = strings.ReplaceAll(bURLParts.Host, ".blob", ".dfs") - bURLParts.BlobName = strings.TrimSuffix(bURLParts.BlobName, "/") // BlobFS doesn't handle folders correctly like this. + if bURLParts.BlobName != "" { + bURLParts.BlobName = strings.TrimSuffix(bURLParts.BlobName, "/") // BlobFS doesn't handle folders correctly like this. + } else { + bURLParts.BlobName = "/" // container level perms MUST have a / + } + // todo: jank, and violates the principle of interfaces fURL := azbfs.NewFileURL(bURLParts.URL(), p.jptm.(*jobPartTransferMgr).jobPartMgr.(*jobPartMgr).secondarySourceProviderPipeline) - return fURL.GetAccessControl(p.jptm.Context()) }