Skip to content

Commit

Permalink
Implement container-level ACL copies (#2049)
Browse files Browse the repository at this point in the history 
* Implement container-level ACL copies

* Prevent AssertNoErr from crashing test suite

* Fix delete functionality & testing

* Grab subdir in a different way

* Fix Build

* Lint
  • Loading branch information
@adreed-msft @nakulkar-msft
adreed-msft authored and nakulkar-msft committed Mar 30, 2023
1 parent 8d1cebd commit 346837a
Show file tree
Hide file tree
Showing 19 changed files with 128 additions and 51 deletions.
8 changes: 4 additions & 4 deletions cmd/copyEnumeratorInit.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ func (cca *CookedCopyCmdArgs) initEnumerator(jobPartOrder common.CopyJobPartOrde
traverser, err = InitResourceTraverser(cca.Source, cca.FromTo.From(), &ctx, &srcCredInfo,
cca.SymlinkHandling, cca.ListOfFilesChannel, cca.Recursive, getRemoteProperties,
cca.IncludeDirectoryStubs, cca.permanentDeleteOption, func(common.EntityType) {}, cca.ListOfVersionIDs,
cca.S2sPreserveBlobTags, common.ESyncHashType.None(), azcopyLogVerbosity.ToPipelineLogLevel(), cca.CpkOptions, nil /* errorChannel */)
cca.S2sPreserveBlobTags, common.ESyncHashType.None(), cca.preservePermissions, azcopyLogVerbosity.ToPipelineLogLevel(), cca.CpkOptions, nil /* errorChannel */)

if err != nil {
return nil, err
Expand Down Expand Up @@ -117,8 +117,8 @@ func (cca *CookedCopyCmdArgs) initEnumerator(jobPartOrder common.CopyJobPartOrde
return nil, errors.New("cannot use --as-subdir=false with a service level destination")
}

// When copying a container directly to a container, strip the top directory
if srcLevel == ELocationLevel.Container() && dstLevel == ELocationLevel.Container() && cca.FromTo.From().IsRemote() && cca.FromTo.To().IsRemote() {
// When copying a container directly to a container, strip the top directory, unless we're attempting to persist permissions.
if srcLevel == ELocationLevel.Container() && dstLevel == ELocationLevel.Container() && cca.FromTo.From().IsRemote() && cca.FromTo.To().IsRemote() && !cca.preservePermissions.IsTruthy() {
cca.StripTopDir = true
}

Expand Down Expand Up @@ -338,7 +338,7 @@ func (cca *CookedCopyCmdArgs) isDestDirectory(dst common.ResourceString, ctx *co

rt, err := InitResourceTraverser(dst, cca.FromTo.To(), ctx, &dstCredInfo, common.ESymlinkHandlingType.Skip(),
nil, false, false, false, common.EPermanentDeleteOption.None(),
func(common.EntityType) {}, cca.ListOfVersionIDs, false, common.ESyncHashType.None(), pipeline.LogNone, cca.CpkOptions, nil /* errorChannel */)
func(common.EntityType) {}, cca.ListOfVersionIDs, false, common.ESyncHashType.None(), cca.preservePermissions, pipeline.LogNone, cca.CpkOptions, nil /* errorChannel */)

if err != nil {
return false
Expand Down
10 changes: 5 additions & 5 deletions cmd/copyEnumeratorInit_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ func (ce *copyEnumeratorSuite) TestValidateSourceDirThatExists(c *chk.C) {

// List
rawBlobURLWithSAS := scenarioHelper{}.getRawBlobURLWithSAS(c, containerName, dirName)
blobTraverser := newBlobTraverser(&rawBlobURLWithSAS, p, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false)
blobTraverser := newBlobTraverser(&rawBlobURLWithSAS, p, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None())

// dir but recursive flag not set - fail
cca := CookedCopyCmdArgs{StripTopDir: false, Recursive: false}
Expand Down Expand Up @@ -78,7 +78,7 @@ func (ce *copyEnumeratorSuite) TestValidateSourceDirDoesNotExist(c *chk.C) {

// List
rawBlobURLWithSAS := scenarioHelper{}.getRawBlobURLWithSAS(c, containerName, dirName)
blobTraverser := newBlobTraverser(&rawBlobURLWithSAS, p, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false)
blobTraverser := newBlobTraverser(&rawBlobURLWithSAS, p, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None())

// dir but recursive flag not set - fail
cca := CookedCopyCmdArgs{StripTopDir: false, Recursive: false}
Expand Down Expand Up @@ -108,7 +108,7 @@ func (ce *copyEnumeratorSuite) TestValidateSourceFileExists(c *chk.C) {

// List
rawBlobURLWithSAS := scenarioHelper{}.getRawBlobURLWithSAS(c, containerName, fileName)
blobTraverser := newBlobTraverser(&rawBlobURLWithSAS, p, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false)
blobTraverser := newBlobTraverser(&rawBlobURLWithSAS, p, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None())

cca := CookedCopyCmdArgs{StripTopDir: false, Recursive: false}
err := cca.validateSourceDir(blobTraverser)
Expand All @@ -131,7 +131,7 @@ func (ce *copyEnumeratorSuite) TestValidateSourceFileDoesNotExist(c *chk.C) {

// List
rawBlobURLWithSAS := scenarioHelper{}.getRawBlobURLWithSAS(c, containerName, fileName)
blobTraverser := newBlobTraverser(&rawBlobURLWithSAS, p, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false)
blobTraverser := newBlobTraverser(&rawBlobURLWithSAS, p, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None())

cca := CookedCopyCmdArgs{StripTopDir: false, Recursive: false}
err := cca.validateSourceDir(blobTraverser)
Expand All @@ -154,7 +154,7 @@ func (ce *copyEnumeratorSuite) TestValidateSourceWithWildCard(c *chk.C) {

// List
rawBlobURLWithSAS := scenarioHelper{}.getRawBlobURLWithSAS(c, containerName, dirName)
blobTraverser := newBlobTraverser(&rawBlobURLWithSAS, p, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false)
blobTraverser := newBlobTraverser(&rawBlobURLWithSAS, p, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None())

// dir but recursive flag not set - fail
cca := CookedCopyCmdArgs{StripTopDir: true, Recursive: false}
Expand Down
2 changes: 1 addition & 1 deletion cmd/list.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -224,7 +224,7 @@ func (cooked cookedListCmdArgs) HandleListContainerCommand() (err error) {

traverser, err := InitResourceTraverser(source, cooked.location, &ctx, &credentialInfo, common.ESymlinkHandlingType.Skip(), nil,
true, false, false, common.EPermanentDeleteOption.None(), func(common.EntityType) {},
nil, false, common.ESyncHashType.None(), pipeline.LogNone, common.CpkOptions{}, nil /* errorChannel */)
nil, false, common.ESyncHashType.None(), common.EPreservePermissionsOption.None(), pipeline.LogNone, common.CpkOptions{}, nil /* errorChannel */)

if err != nil {
return fmt.Errorf("failed to initialize traverser: %s", err.Error())
Expand Down
2 changes: 1 addition & 1 deletion cmd/removeEnumerator.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ func newRemoveEnumerator(cca *CookedCopyCmdArgs) (enumerator *CopyEnumerator, er
sourceTraverser, err = InitResourceTraverser(cca.Source, cca.FromTo.From(), &ctx, &cca.credentialInfo,
common.ESymlinkHandlingType.Skip(), cca.ListOfFilesChannel, cca.Recursive, false, cca.IncludeDirectoryStubs,
cca.permanentDeleteOption, func(common.EntityType) {}, cca.ListOfVersionIDs, false,
common.ESyncHashType.None(), azcopyLogVerbosity.ToPipelineLogLevel(), cca.CpkOptions, nil /* errorChannel */)
common.ESyncHashType.None(), common.EPreservePermissionsOption.None(), azcopyLogVerbosity.ToPipelineLogLevel(), cca.CpkOptions, nil /* errorChannel */)

// report failure to create traverser
if err != nil {
Expand Down
2 changes: 1 addition & 1 deletion cmd/setPropertiesEnumerator.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ func setPropertiesEnumerator(cca *CookedCopyCmdArgs) (enumerator *CopyEnumerator
common.ESymlinkHandlingType.Preserve(), // preserve because we want to index all blobs, including symlink blobs
cca.ListOfFilesChannel, cca.Recursive, false, cca.IncludeDirectoryStubs,
cca.permanentDeleteOption, func(common.EntityType) {}, cca.ListOfVersionIDs, false,
common.ESyncHashType.None(), azcopyLogVerbosity.ToPipelineLogLevel(), cca.CpkOptions, nil /* errorChannel */)
common.ESyncHashType.None(), common.EPreservePermissionsOption.None(), azcopyLogVerbosity.ToPipelineLogLevel(), cca.CpkOptions, nil /* errorChannel */)

// report failure to create traverser
if err != nil {
Expand Down
4 changes: 2 additions & 2 deletions cmd/syncEnumerator.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@ func (cca *cookedSyncCmdArgs) initEnumerator(ctx context.Context) (enumerator *s
if entityType == common.EEntityType.File() {
atomic.AddUint64(&cca.atomicSourceFilesScanned, 1)
}
}, nil, cca.s2sPreserveBlobTags, cca.compareHash, azcopyLogVerbosity.ToPipelineLogLevel(), cca.cpkOptions, nil /* errorChannel */)
}, nil, cca.s2sPreserveBlobTags, cca.compareHash, cca.preservePermissions, azcopyLogVerbosity.ToPipelineLogLevel(), cca.cpkOptions, nil /* errorChannel */)

if err != nil {
return nil, err
Expand All @@ -86,7 +86,7 @@ func (cca *cookedSyncCmdArgs) initEnumerator(ctx context.Context) (enumerator *s
if entityType == common.EEntityType.File() {
atomic.AddUint64(&cca.atomicDestinationFilesScanned, 1)
}
}, nil, cca.s2sPreserveBlobTags, cca.compareHash, azcopyLogVerbosity.ToPipelineLogLevel(), cca.cpkOptions, nil /* errorChannel */)
}, nil, cca.s2sPreserveBlobTags, cca.compareHash, cca.preservePermissions, azcopyLogVerbosity.ToPipelineLogLevel(), cca.cpkOptions, nil /* errorChannel */)
if err != nil {
return nil, err
}
Expand Down
10 changes: 5 additions & 5 deletions cmd/zc_enumerator.go
100644 → 100755
View file
Original file line number Diff line number Diff line change
Expand Up @@ -332,7 +332,7 @@ type enumerationCounterFunc func(entityType common.EntityType)
func InitResourceTraverser(resource common.ResourceString, location common.Location, ctx *context.Context,
credential *common.CredentialInfo, symlinkHandling common.SymlinkHandlingType, listOfFilesChannel chan string, recursive, getProperties,
includeDirectoryStubs bool, permanentDeleteOption common.PermanentDeleteOption, incrementEnumerationCounter enumerationCounterFunc, listOfVersionIds chan string,
s2sPreserveBlobTags bool, syncHashType common.SyncHashType, logLevel pipeline.LogLevel, cpkOptions common.CpkOptions, errorChannel chan ErrorFileInfo) (ResourceTraverser, error) {
s2sPreserveBlobTags bool, syncHashType common.SyncHashType, preservePermissions common.PreservePermissionsOption, logLevel pipeline.LogLevel, cpkOptions common.CpkOptions, errorChannel chan ErrorFileInfo) (ResourceTraverser, error) {
var output ResourceTraverser
var p *pipeline.Pipeline

Expand Down Expand Up @@ -380,7 +380,7 @@ func InitResourceTraverser(resource common.ResourceString, location common.Locat
}

output = newListTraverser(resource, location, credential, ctx, recursive, symlinkHandling, getProperties,
listOfFilesChannel, includeDirectoryStubs, incrementEnumerationCounter, s2sPreserveBlobTags, logLevel, cpkOptions)
listOfFilesChannel, includeDirectoryStubs, incrementEnumerationCounter, s2sPreserveBlobTags, logLevel, cpkOptions, syncHashType, preservePermissions)
return output, nil
}

Expand Down Expand Up @@ -408,7 +408,7 @@ func InitResourceTraverser(resource common.ResourceString, location common.Locat

baseResource := resource.CloneWithValue(cleanLocalPath(basePath))
output = newListTraverser(baseResource, location, nil, nil, recursive, symlinkHandling, getProperties,
globChan, includeDirectoryStubs, incrementEnumerationCounter, s2sPreserveBlobTags, logLevel, cpkOptions)
globChan, includeDirectoryStubs, incrementEnumerationCounter, s2sPreserveBlobTags, logLevel, cpkOptions, syncHashType, preservePermissions)
} else {
if ctx != nil {
output = newLocalTraverser(*ctx, resource.ValueLocal(), recursive, symlinkHandling, syncHashType, incrementEnumerationCounter, errorChannel)
Expand Down Expand Up @@ -443,11 +443,11 @@ func InitResourceTraverser(resource common.ResourceString, location common.Locat
return nil, errors.New(accountTraversalInherentlyRecursiveError)
}

output = newBlobAccountTraverser(resourceURL, *p, *ctx, includeDirectoryStubs, incrementEnumerationCounter, s2sPreserveBlobTags, cpkOptions)
output = newBlobAccountTraverser(resourceURL, *p, *ctx, includeDirectoryStubs, incrementEnumerationCounter, s2sPreserveBlobTags, cpkOptions, preservePermissions)
} else if listOfVersionIds != nil {
output = newBlobVersionsTraverser(resourceURL, *p, *ctx, recursive, includeDirectoryStubs, incrementEnumerationCounter, listOfVersionIds, cpkOptions)
} else {
output = newBlobTraverser(resourceURL, *p, *ctx, recursive, includeDirectoryStubs, incrementEnumerationCounter, s2sPreserveBlobTags, cpkOptions, includeDeleted, includeSnapshot, includeVersion)
output = newBlobTraverser(resourceURL, *p, *ctx, recursive, includeDirectoryStubs, incrementEnumerationCounter, s2sPreserveBlobTags, cpkOptions, includeDeleted, includeSnapshot, includeVersion, preservePermissions)
}
case common.ELocation.File():
resourceURL, err := resource.FullURL()
Expand Down
36 changes: 34 additions & 2 deletions cmd/zc_traverser_blob.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ import (
"fmt"
"net/url"
"strings"
"time"

"github.com/Azure/azure-storage-azcopy/v10/common/parallel"

Expand Down Expand Up @@ -56,6 +57,8 @@ type blobTraverser struct {

cpkOptions common.CpkOptions

preservePermissions common.PreservePermissionsOption

includeDeleted bool

includeSnapshot bool
Expand Down Expand Up @@ -222,7 +225,7 @@ func (t *blobTraverser) Traverse(preprocessor objectMorpher, processor objectPro
}
}
if t.incrementEnumerationCounter != nil {
t.incrementEnumerationCounter(common.EEntityType.File())
t.incrementEnumerationCounter(storedObject.entityType)
}

err := processIfPassedFilters(filters, storedObject, processor)
Expand All @@ -232,6 +235,34 @@ func (t *blobTraverser) Traverse(preprocessor objectMorpher, processor objectPro
if !t.includeDeleted && (isBlob || err != nil) {
return err
}
} else if blobUrlParts.BlobName == "" && t.preservePermissions.IsTruthy() {
// if the root is a container and we're copying "folders", we should persist the ACLs there too.
if azcopyScanningLogger != nil {
azcopyScanningLogger.Log(pipeline.LogDebug, "Detected the root as a container.")
}

storedObject := newStoredObject(
preprocessor,
"",
"",
common.EEntityType.Folder(),
time.Now(),
0,
noContentProps,
noBlobProps,
common.Metadata{},
blobUrlParts.ContainerName,
)

if t.incrementEnumerationCounter != nil {
t.incrementEnumerationCounter(common.EEntityType.Folder())
}

err := processIfPassedFilters(filters, storedObject, processor)
_, err = getProcessingError(err)
if err != nil {
return err
}
}

// get the container URL so that we can list the blobs
Expand Down Expand Up @@ -487,7 +518,7 @@ func (t *blobTraverser) serialList(containerURL azblob.ContainerURL, containerNa
return nil
}

func newBlobTraverser(rawURL *url.URL, p pipeline.Pipeline, ctx context.Context, recursive, includeDirectoryStubs bool, incrementEnumerationCounter enumerationCounterFunc, s2sPreserveSourceTags bool, cpkOptions common.CpkOptions, includeDeleted, includeSnapshot, includeVersion bool) (t *blobTraverser) {
func newBlobTraverser(rawURL *url.URL, p pipeline.Pipeline, ctx context.Context, recursive, includeDirectoryStubs bool, incrementEnumerationCounter enumerationCounterFunc, s2sPreserveSourceTags bool, cpkOptions common.CpkOptions, includeDeleted, includeSnapshot, includeVersion bool, preservePermissions common.PreservePermissionsOption) (t *blobTraverser) {
t = &blobTraverser{
rawURL: rawURL,
p: p,
Expand All @@ -501,6 +532,7 @@ func newBlobTraverser(rawURL *url.URL, p pipeline.Pipeline, ctx context.Context,
includeDeleted: includeDeleted,
includeSnapshot: includeSnapshot,
includeVersion: includeVersion,
preservePermissions: preservePermissions,
}

disableHierarchicalScanning := strings.ToLower(glcm.GetEnvironmentVariable(common.EEnvironmentVariable.DisableHierarchicalScanning()))
Expand Down
6 changes: 4 additions & 2 deletions cmd/zc_traverser_blob_account.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ type blobAccountTraverser struct {
s2sPreserveSourceTags bool

cpkOptions common.CpkOptions
preservePermissions common.PreservePermissionsOption
}

func (t *blobAccountTraverser) IsDirectory(_ bool) (bool, error) {
Expand Down Expand Up @@ -100,7 +101,7 @@ func (t *blobAccountTraverser) Traverse(preprocessor objectMorpher, processor ob

for _, v := range cList {
containerURL := t.accountURL.NewContainerURL(v).URL()
containerTraverser := newBlobTraverser(&containerURL, t.p, t.ctx, true, t.includeDirectoryStubs, t.incrementEnumerationCounter, t.s2sPreserveSourceTags, t.cpkOptions, false, false, false)
containerTraverser := newBlobTraverser(&containerURL, t.p, t.ctx, true, t.includeDirectoryStubs, t.incrementEnumerationCounter, t.s2sPreserveSourceTags, t.cpkOptions, false, false, false, t.preservePermissions)

preprocessorForThisChild := preprocessor.FollowedBy(newContainerDecorator(v))

Expand All @@ -115,7 +116,7 @@ func (t *blobAccountTraverser) Traverse(preprocessor objectMorpher, processor ob
return nil
}

func newBlobAccountTraverser(rawURL *url.URL, p pipeline.Pipeline, ctx context.Context, includeDirectoryStubs bool, incrementEnumerationCounter enumerationCounterFunc, s2sPreserveSourceTags bool, cpkOptions common.CpkOptions) (t *blobAccountTraverser) {
func newBlobAccountTraverser(rawURL *url.URL, p pipeline.Pipeline, ctx context.Context, includeDirectoryStubs bool, incrementEnumerationCounter enumerationCounterFunc, s2sPreserveSourceTags bool, cpkOptions common.CpkOptions, preservePermissions common.PreservePermissionsOption) (t *blobAccountTraverser) {
bURLParts := azblob.NewBlobURLParts(*rawURL)
cPattern := bURLParts.ContainerName

Expand All @@ -133,6 +134,7 @@ func newBlobAccountTraverser(rawURL *url.URL, p pipeline.Pipeline, ctx context.C
includeDirectoryStubs: includeDirectoryStubs,
s2sPreserveSourceTags: s2sPreserveSourceTags,
cpkOptions: cpkOptions,
preservePermissions: preservePermissions,
}

return
Expand Down
5 changes: 2 additions & 3 deletions cmd/zc_traverser_list.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -92,8 +92,7 @@ func (l *listTraverser) Traverse(preprocessor objectMorpher, processor objectPro
func newListTraverser(parent common.ResourceString, parentType common.Location, credential *common.CredentialInfo,
ctx *context.Context, recursive bool, handleSymlinks common.SymlinkHandlingType, getProperties bool, listChan chan string,
includeDirectoryStubs bool, incrementEnumerationCounter enumerationCounterFunc, s2sPreserveBlobTags bool,
logLevel pipeline.LogLevel, cpkOptions common.CpkOptions) ResourceTraverser {

logLevel pipeline.LogLevel, cpkOptions common.CpkOptions, syncHashType common.SyncHashType, preservePermissions common.PreservePermissionsOption) ResourceTraverser {
traverserGenerator := func(relativeChildPath string) (ResourceTraverser, error) {
source := parent.Clone()
if parentType != common.ELocation.Local() {
Expand All @@ -109,7 +108,7 @@ func newListTraverser(parent common.ResourceString, parentType common.Location,
// Construct a traverser that goes through the child
traverser, err := InitResourceTraverser(source, parentType, ctx, credential, handleSymlinks,
nil, recursive, getProperties, includeDirectoryStubs, common.EPermanentDeleteOption.None(), incrementEnumerationCounter,
nil, s2sPreserveBlobTags, common.ESyncHashType.None(), logLevel, cpkOptions, nil /* errorChannel */)
nil, s2sPreserveBlobTags, syncHashType, preservePermissions, logLevel, cpkOptions, nil /* errorChannel */)
if err != nil {
return nil, err
}
Expand Down
4 changes: 2 additions & 2 deletions cmd/zt_copy_blob_download_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -174,7 +174,7 @@ func (s *cmdIntegrationSuite) TestDownloadAccount(c *chk.C) {

// Traverse the account ahead of time and determine the relative paths for testing.
relPaths := make([]string, 0) // Use a map for easy lookup
blobTraverser := newBlobAccountTraverser(&rawBSU, p, ctx, false, func(common.EntityType) {}, false, common.CpkOptions{})
blobTraverser := newBlobAccountTraverser(&rawBSU, p, ctx, false, func(common.EntityType) {}, false, common.CpkOptions{}, common.EPreservePermissionsOption.None())
processor := func(object StoredObject) error {
// Append the container name to the relative path
relPath := "/" + object.ContainerName + "/" + object.relativePath
Expand Down Expand Up @@ -222,7 +222,7 @@ func (s *cmdIntegrationSuite) TestDownloadAccountWildcard(c *chk.C) {

// Traverse the account ahead of time and determine the relative paths for testing.
relPaths := make([]string, 0) // Use a map for easy lookup
blobTraverser := newBlobAccountTraverser(&rawBSU, p, ctx, false, func(common.EntityType) {}, false, common.CpkOptions{})
blobTraverser := newBlobAccountTraverser(&rawBSU, p, ctx, false, func(common.EntityType) {}, false, common.CpkOptions{}, common.EPreservePermissionsOption.None())
processor := func(object StoredObject) error {
// Append the container name to the relative path
relPath := "/" + object.ContainerName + "/" + object.relativePath
Expand Down

0 comments on commit 346837a

Please sign in to comment.