Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Implement container-level ACL copies #2049

Merged
merged 7 commits into from
Mar 9, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions cmd/copyEnumeratorInit.go
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ func (cca *CookedCopyCmdArgs) initEnumerator(jobPartOrder common.CopyJobPartOrde
traverser, err = InitResourceTraverser(cca.Source, cca.FromTo.From(), &ctx, &srcCredInfo,
cca.SymlinkHandling, cca.ListOfFilesChannel, cca.Recursive, getRemoteProperties,
cca.IncludeDirectoryStubs, cca.permanentDeleteOption, func(common.EntityType) {}, cca.ListOfVersionIDs,
cca.S2sPreserveBlobTags, common.ESyncHashType.None(), azcopyLogVerbosity.ToPipelineLogLevel(), cca.CpkOptions, nil /* errorChannel */)
cca.S2sPreserveBlobTags, common.ESyncHashType.None(), cca.preservePermissions, azcopyLogVerbosity.ToPipelineLogLevel(), cca.CpkOptions, nil /* errorChannel */)

if err != nil {
return nil, err
Expand Down Expand Up @@ -117,8 +117,8 @@ func (cca *CookedCopyCmdArgs) initEnumerator(jobPartOrder common.CopyJobPartOrde
return nil, errors.New("cannot use --as-subdir=false with a service level destination")
}

// When copying a container directly to a container, strip the top directory
if srcLevel == ELocationLevel.Container() && dstLevel == ELocationLevel.Container() && cca.FromTo.From().IsRemote() && cca.FromTo.To().IsRemote() {
// When copying a container directly to a container, strip the top directory, unless we're attempting to persist permissions.
if srcLevel == ELocationLevel.Container() && dstLevel == ELocationLevel.Container() && cca.FromTo.From().IsRemote() && cca.FromTo.To().IsRemote() && !cca.preservePermissions.IsTruthy() {
cca.StripTopDir = true
}

Expand Down Expand Up @@ -338,7 +338,7 @@ func (cca *CookedCopyCmdArgs) isDestDirectory(dst common.ResourceString, ctx *co

rt, err := InitResourceTraverser(dst, cca.FromTo.To(), ctx, &dstCredInfo, common.ESymlinkHandlingType.Skip(),
nil, false, false, false, common.EPermanentDeleteOption.None(),
func(common.EntityType) {}, cca.ListOfVersionIDs, false, common.ESyncHashType.None(), pipeline.LogNone, cca.CpkOptions, nil /* errorChannel */)
func(common.EntityType) {}, cca.ListOfVersionIDs, false, common.ESyncHashType.None(), cca.preservePermissions, pipeline.LogNone, cca.CpkOptions, nil /* errorChannel */)

if err != nil {
return false
Expand Down
10 changes: 5 additions & 5 deletions cmd/copyEnumeratorInit_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ func (ce *copyEnumeratorSuite) TestValidateSourceDirThatExists(c *chk.C) {

// List
rawBlobURLWithSAS := scenarioHelper{}.getRawBlobURLWithSAS(c, containerName, dirName)
blobTraverser := newBlobTraverser(&rawBlobURLWithSAS, p, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false)
blobTraverser := newBlobTraverser(&rawBlobURLWithSAS, p, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None())

// dir but recursive flag not set - fail
cca := CookedCopyCmdArgs{StripTopDir: false, Recursive: false}
Expand Down Expand Up @@ -78,7 +78,7 @@ func (ce *copyEnumeratorSuite) TestValidateSourceDirDoesNotExist(c *chk.C) {

// List
rawBlobURLWithSAS := scenarioHelper{}.getRawBlobURLWithSAS(c, containerName, dirName)
blobTraverser := newBlobTraverser(&rawBlobURLWithSAS, p, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false)
blobTraverser := newBlobTraverser(&rawBlobURLWithSAS, p, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None())

// dir but recursive flag not set - fail
cca := CookedCopyCmdArgs{StripTopDir: false, Recursive: false}
Expand Down Expand Up @@ -108,7 +108,7 @@ func (ce *copyEnumeratorSuite) TestValidateSourceFileExists(c *chk.C) {

// List
rawBlobURLWithSAS := scenarioHelper{}.getRawBlobURLWithSAS(c, containerName, fileName)
blobTraverser := newBlobTraverser(&rawBlobURLWithSAS, p, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false)
blobTraverser := newBlobTraverser(&rawBlobURLWithSAS, p, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None())

cca := CookedCopyCmdArgs{StripTopDir: false, Recursive: false}
err := cca.validateSourceDir(blobTraverser)
Expand All @@ -131,7 +131,7 @@ func (ce *copyEnumeratorSuite) TestValidateSourceFileDoesNotExist(c *chk.C) {

// List
rawBlobURLWithSAS := scenarioHelper{}.getRawBlobURLWithSAS(c, containerName, fileName)
blobTraverser := newBlobTraverser(&rawBlobURLWithSAS, p, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false)
blobTraverser := newBlobTraverser(&rawBlobURLWithSAS, p, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None())

cca := CookedCopyCmdArgs{StripTopDir: false, Recursive: false}
err := cca.validateSourceDir(blobTraverser)
Expand All @@ -154,7 +154,7 @@ func (ce *copyEnumeratorSuite) TestValidateSourceWithWildCard(c *chk.C) {

// List
rawBlobURLWithSAS := scenarioHelper{}.getRawBlobURLWithSAS(c, containerName, dirName)
blobTraverser := newBlobTraverser(&rawBlobURLWithSAS, p, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false)
blobTraverser := newBlobTraverser(&rawBlobURLWithSAS, p, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None())

// dir but recursive flag not set - fail
cca := CookedCopyCmdArgs{StripTopDir: true, Recursive: false}
Expand Down
2 changes: 1 addition & 1 deletion cmd/list.go
Original file line number Diff line number Diff line change
Expand Up @@ -224,7 +224,7 @@ func (cooked cookedListCmdArgs) HandleListContainerCommand() (err error) {

traverser, err := InitResourceTraverser(source, cooked.location, &ctx, &credentialInfo, common.ESymlinkHandlingType.Skip(), nil,
true, false, false, common.EPermanentDeleteOption.None(), func(common.EntityType) {},
nil, false, common.ESyncHashType.None(), pipeline.LogNone, common.CpkOptions{}, nil /* errorChannel */)
nil, false, common.ESyncHashType.None(), common.EPreservePermissionsOption.None(), pipeline.LogNone, common.CpkOptions{}, nil /* errorChannel */)

if err != nil {
return fmt.Errorf("failed to initialize traverser: %s", err.Error())
Expand Down
2 changes: 1 addition & 1 deletion cmd/removeEnumerator.go
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ func newRemoveEnumerator(cca *CookedCopyCmdArgs) (enumerator *CopyEnumerator, er
sourceTraverser, err = InitResourceTraverser(cca.Source, cca.FromTo.From(), &ctx, &cca.credentialInfo,
common.ESymlinkHandlingType.Skip(), cca.ListOfFilesChannel, cca.Recursive, false, cca.IncludeDirectoryStubs,
cca.permanentDeleteOption, func(common.EntityType) {}, cca.ListOfVersionIDs, false,
common.ESyncHashType.None(), azcopyLogVerbosity.ToPipelineLogLevel(), cca.CpkOptions, nil /* errorChannel */)
common.ESyncHashType.None(), common.EPreservePermissionsOption.None(), azcopyLogVerbosity.ToPipelineLogLevel(), cca.CpkOptions, nil /* errorChannel */)

// report failure to create traverser
if err != nil {
Expand Down
2 changes: 1 addition & 1 deletion cmd/setPropertiesEnumerator.go
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ func setPropertiesEnumerator(cca *CookedCopyCmdArgs) (enumerator *CopyEnumerator
common.ESymlinkHandlingType.Preserve(), // preserve because we want to index all blobs, including symlink blobs
cca.ListOfFilesChannel, cca.Recursive, false, cca.IncludeDirectoryStubs,
cca.permanentDeleteOption, func(common.EntityType) {}, cca.ListOfVersionIDs, false,
common.ESyncHashType.None(), azcopyLogVerbosity.ToPipelineLogLevel(), cca.CpkOptions, nil /* errorChannel */)
common.ESyncHashType.None(), common.EPreservePermissionsOption.None(), azcopyLogVerbosity.ToPipelineLogLevel(), cca.CpkOptions, nil /* errorChannel */)

// report failure to create traverser
if err != nil {
Expand Down
4 changes: 2 additions & 2 deletions cmd/syncEnumerator.go
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@ func (cca *cookedSyncCmdArgs) initEnumerator(ctx context.Context) (enumerator *s
if entityType == common.EEntityType.File() {
atomic.AddUint64(&cca.atomicSourceFilesScanned, 1)
}
}, nil, cca.s2sPreserveBlobTags, cca.compareHash, azcopyLogVerbosity.ToPipelineLogLevel(), cca.cpkOptions, nil /* errorChannel */)
}, nil, cca.s2sPreserveBlobTags, cca.compareHash, cca.preservePermissions, azcopyLogVerbosity.ToPipelineLogLevel(), cca.cpkOptions, nil /* errorChannel */)

if err != nil {
return nil, err
Expand All @@ -86,7 +86,7 @@ func (cca *cookedSyncCmdArgs) initEnumerator(ctx context.Context) (enumerator *s
if entityType == common.EEntityType.File() {
atomic.AddUint64(&cca.atomicDestinationFilesScanned, 1)
}
}, nil, cca.s2sPreserveBlobTags, cca.compareHash, azcopyLogVerbosity.ToPipelineLogLevel(), cca.cpkOptions, nil /* errorChannel */)
}, nil, cca.s2sPreserveBlobTags, cca.compareHash, cca.preservePermissions, azcopyLogVerbosity.ToPipelineLogLevel(), cca.cpkOptions, nil /* errorChannel */)
if err != nil {
return nil, err
}
Expand Down
10 changes: 5 additions & 5 deletions cmd/zc_enumerator.go
100644 → 100755
Original file line number Diff line number Diff line change
Expand Up @@ -332,7 +332,7 @@ type enumerationCounterFunc func(entityType common.EntityType)
func InitResourceTraverser(resource common.ResourceString, location common.Location, ctx *context.Context,
credential *common.CredentialInfo, symlinkHandling common.SymlinkHandlingType, listOfFilesChannel chan string, recursive, getProperties,
includeDirectoryStubs bool, permanentDeleteOption common.PermanentDeleteOption, incrementEnumerationCounter enumerationCounterFunc, listOfVersionIds chan string,
s2sPreserveBlobTags bool, syncHashType common.SyncHashType, logLevel pipeline.LogLevel, cpkOptions common.CpkOptions, errorChannel chan ErrorFileInfo) (ResourceTraverser, error) {
s2sPreserveBlobTags bool, syncHashType common.SyncHashType, preservePermissions common.PreservePermissionsOption, logLevel pipeline.LogLevel, cpkOptions common.CpkOptions, errorChannel chan ErrorFileInfo) (ResourceTraverser, error) {
var output ResourceTraverser
var p *pipeline.Pipeline

Expand Down Expand Up @@ -380,7 +380,7 @@ func InitResourceTraverser(resource common.ResourceString, location common.Locat
}

output = newListTraverser(resource, location, credential, ctx, recursive, symlinkHandling, getProperties,
listOfFilesChannel, includeDirectoryStubs, incrementEnumerationCounter, s2sPreserveBlobTags, logLevel, cpkOptions)
listOfFilesChannel, includeDirectoryStubs, incrementEnumerationCounter, s2sPreserveBlobTags, logLevel, cpkOptions, syncHashType, preservePermissions)
return output, nil
}

Expand Down Expand Up @@ -408,7 +408,7 @@ func InitResourceTraverser(resource common.ResourceString, location common.Locat

baseResource := resource.CloneWithValue(cleanLocalPath(basePath))
output = newListTraverser(baseResource, location, nil, nil, recursive, symlinkHandling, getProperties,
globChan, includeDirectoryStubs, incrementEnumerationCounter, s2sPreserveBlobTags, logLevel, cpkOptions)
globChan, includeDirectoryStubs, incrementEnumerationCounter, s2sPreserveBlobTags, logLevel, cpkOptions, syncHashType, preservePermissions)
} else {
if ctx != nil {
output = newLocalTraverser(*ctx, resource.ValueLocal(), recursive, symlinkHandling, syncHashType, incrementEnumerationCounter, errorChannel)
Expand Down Expand Up @@ -443,11 +443,11 @@ func InitResourceTraverser(resource common.ResourceString, location common.Locat
return nil, errors.New(accountTraversalInherentlyRecursiveError)
}

output = newBlobAccountTraverser(resourceURL, *p, *ctx, includeDirectoryStubs, incrementEnumerationCounter, s2sPreserveBlobTags, cpkOptions)
output = newBlobAccountTraverser(resourceURL, *p, *ctx, includeDirectoryStubs, incrementEnumerationCounter, s2sPreserveBlobTags, cpkOptions, preservePermissions)
} else if listOfVersionIds != nil {
output = newBlobVersionsTraverser(resourceURL, *p, *ctx, recursive, includeDirectoryStubs, incrementEnumerationCounter, listOfVersionIds, cpkOptions)
} else {
output = newBlobTraverser(resourceURL, *p, *ctx, recursive, includeDirectoryStubs, incrementEnumerationCounter, s2sPreserveBlobTags, cpkOptions, includeDeleted, includeSnapshot, includeVersion)
output = newBlobTraverser(resourceURL, *p, *ctx, recursive, includeDirectoryStubs, incrementEnumerationCounter, s2sPreserveBlobTags, cpkOptions, includeDeleted, includeSnapshot, includeVersion, preservePermissions)
}
case common.ELocation.File():
resourceURL, err := resource.FullURL()
Expand Down
36 changes: 34 additions & 2 deletions cmd/zc_traverser_blob.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ import (
"fmt"
"net/url"
"strings"
"time"

"github.com/Azure/azure-storage-azcopy/v10/common/parallel"

Expand Down Expand Up @@ -56,6 +57,8 @@ type blobTraverser struct {

cpkOptions common.CpkOptions

preservePermissions common.PreservePermissionsOption

includeDeleted bool

includeSnapshot bool
Expand Down Expand Up @@ -222,7 +225,7 @@ func (t *blobTraverser) Traverse(preprocessor objectMorpher, processor objectPro
}
}
if t.incrementEnumerationCounter != nil {
t.incrementEnumerationCounter(common.EEntityType.File())
t.incrementEnumerationCounter(storedObject.entityType)
}

err := processIfPassedFilters(filters, storedObject, processor)
Expand All @@ -232,6 +235,34 @@ func (t *blobTraverser) Traverse(preprocessor objectMorpher, processor objectPro
if !t.includeDeleted && (isBlob || err != nil) {
return err
}
} else if blobUrlParts.BlobName == "" && t.preservePermissions.IsTruthy() {
// if the root is a container and we're copying "folders", we should persist the ACLs there too.
if azcopyScanningLogger != nil {
azcopyScanningLogger.Log(pipeline.LogDebug, "Detected the root as a container.")
}

storedObject := newStoredObject(
preprocessor,
"",
"",
common.EEntityType.Folder(),
time.Now(),
0,
noContentProps,
noBlobProps,
common.Metadata{},
blobUrlParts.ContainerName,
)

if t.incrementEnumerationCounter != nil {
t.incrementEnumerationCounter(common.EEntityType.Folder())
}

err := processIfPassedFilters(filters, storedObject, processor)
_, err = getProcessingError(err)
if err != nil {
return err
}
}

// get the container URL so that we can list the blobs
Expand Down Expand Up @@ -487,7 +518,7 @@ func (t *blobTraverser) serialList(containerURL azblob.ContainerURL, containerNa
return nil
}

func newBlobTraverser(rawURL *url.URL, p pipeline.Pipeline, ctx context.Context, recursive, includeDirectoryStubs bool, incrementEnumerationCounter enumerationCounterFunc, s2sPreserveSourceTags bool, cpkOptions common.CpkOptions, includeDeleted, includeSnapshot, includeVersion bool) (t *blobTraverser) {
func newBlobTraverser(rawURL *url.URL, p pipeline.Pipeline, ctx context.Context, recursive, includeDirectoryStubs bool, incrementEnumerationCounter enumerationCounterFunc, s2sPreserveSourceTags bool, cpkOptions common.CpkOptions, includeDeleted, includeSnapshot, includeVersion bool, preservePermissions common.PreservePermissionsOption) (t *blobTraverser) {
t = &blobTraverser{
rawURL: rawURL,
p: p,
Expand All @@ -501,6 +532,7 @@ func newBlobTraverser(rawURL *url.URL, p pipeline.Pipeline, ctx context.Context,
includeDeleted: includeDeleted,
includeSnapshot: includeSnapshot,
includeVersion: includeVersion,
preservePermissions: preservePermissions,
}

disableHierarchicalScanning := strings.ToLower(glcm.GetEnvironmentVariable(common.EEnvironmentVariable.DisableHierarchicalScanning()))
Expand Down
6 changes: 4 additions & 2 deletions cmd/zc_traverser_blob_account.go
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ type blobAccountTraverser struct {
s2sPreserveSourceTags bool

cpkOptions common.CpkOptions
preservePermissions common.PreservePermissionsOption
}

func (t *blobAccountTraverser) IsDirectory(_ bool) (bool, error) {
Expand Down Expand Up @@ -100,7 +101,7 @@ func (t *blobAccountTraverser) Traverse(preprocessor objectMorpher, processor ob

for _, v := range cList {
containerURL := t.accountURL.NewContainerURL(v).URL()
containerTraverser := newBlobTraverser(&containerURL, t.p, t.ctx, true, t.includeDirectoryStubs, t.incrementEnumerationCounter, t.s2sPreserveSourceTags, t.cpkOptions, false, false, false)
containerTraverser := newBlobTraverser(&containerURL, t.p, t.ctx, true, t.includeDirectoryStubs, t.incrementEnumerationCounter, t.s2sPreserveSourceTags, t.cpkOptions, false, false, false, t.preservePermissions)

preprocessorForThisChild := preprocessor.FollowedBy(newContainerDecorator(v))

Expand All @@ -115,7 +116,7 @@ func (t *blobAccountTraverser) Traverse(preprocessor objectMorpher, processor ob
return nil
}

func newBlobAccountTraverser(rawURL *url.URL, p pipeline.Pipeline, ctx context.Context, includeDirectoryStubs bool, incrementEnumerationCounter enumerationCounterFunc, s2sPreserveSourceTags bool, cpkOptions common.CpkOptions) (t *blobAccountTraverser) {
func newBlobAccountTraverser(rawURL *url.URL, p pipeline.Pipeline, ctx context.Context, includeDirectoryStubs bool, incrementEnumerationCounter enumerationCounterFunc, s2sPreserveSourceTags bool, cpkOptions common.CpkOptions, preservePermissions common.PreservePermissionsOption) (t *blobAccountTraverser) {
bURLParts := azblob.NewBlobURLParts(*rawURL)
cPattern := bURLParts.ContainerName

Expand All @@ -133,6 +134,7 @@ func newBlobAccountTraverser(rawURL *url.URL, p pipeline.Pipeline, ctx context.C
includeDirectoryStubs: includeDirectoryStubs,
s2sPreserveSourceTags: s2sPreserveSourceTags,
cpkOptions: cpkOptions,
preservePermissions: preservePermissions,
}

return
Expand Down
5 changes: 2 additions & 3 deletions cmd/zc_traverser_list.go
Original file line number Diff line number Diff line change
Expand Up @@ -92,8 +92,7 @@ func (l *listTraverser) Traverse(preprocessor objectMorpher, processor objectPro
func newListTraverser(parent common.ResourceString, parentType common.Location, credential *common.CredentialInfo,
ctx *context.Context, recursive bool, handleSymlinks common.SymlinkHandlingType, getProperties bool, listChan chan string,
includeDirectoryStubs bool, incrementEnumerationCounter enumerationCounterFunc, s2sPreserveBlobTags bool,
logLevel pipeline.LogLevel, cpkOptions common.CpkOptions) ResourceTraverser {

logLevel pipeline.LogLevel, cpkOptions common.CpkOptions, syncHashType common.SyncHashType, preservePermissions common.PreservePermissionsOption) ResourceTraverser {
traverserGenerator := func(relativeChildPath string) (ResourceTraverser, error) {
source := parent.Clone()
if parentType != common.ELocation.Local() {
Expand All @@ -109,7 +108,7 @@ func newListTraverser(parent common.ResourceString, parentType common.Location,
// Construct a traverser that goes through the child
traverser, err := InitResourceTraverser(source, parentType, ctx, credential, handleSymlinks,
nil, recursive, getProperties, includeDirectoryStubs, common.EPermanentDeleteOption.None(), incrementEnumerationCounter,
nil, s2sPreserveBlobTags, common.ESyncHashType.None(), logLevel, cpkOptions, nil /* errorChannel */)
nil, s2sPreserveBlobTags, syncHashType, preservePermissions, logLevel, cpkOptions, nil /* errorChannel */)
if err != nil {
return nil, err
}
Expand Down
4 changes: 2 additions & 2 deletions cmd/zt_copy_blob_download_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -174,7 +174,7 @@ func (s *cmdIntegrationSuite) TestDownloadAccount(c *chk.C) {

// Traverse the account ahead of time and determine the relative paths for testing.
relPaths := make([]string, 0) // Use a map for easy lookup
blobTraverser := newBlobAccountTraverser(&rawBSU, p, ctx, false, func(common.EntityType) {}, false, common.CpkOptions{})
blobTraverser := newBlobAccountTraverser(&rawBSU, p, ctx, false, func(common.EntityType) {}, false, common.CpkOptions{}, common.EPreservePermissionsOption.None())
processor := func(object StoredObject) error {
// Append the container name to the relative path
relPath := "/" + object.ContainerName + "/" + object.relativePath
Expand Down Expand Up @@ -222,7 +222,7 @@ func (s *cmdIntegrationSuite) TestDownloadAccountWildcard(c *chk.C) {

// Traverse the account ahead of time and determine the relative paths for testing.
relPaths := make([]string, 0) // Use a map for easy lookup
blobTraverser := newBlobAccountTraverser(&rawBSU, p, ctx, false, func(common.EntityType) {}, false, common.CpkOptions{})
blobTraverser := newBlobAccountTraverser(&rawBSU, p, ctx, false, func(common.EntityType) {}, false, common.CpkOptions{}, common.EPreservePermissionsOption.None())
processor := func(object StoredObject) error {
// Append the container name to the relative path
relPath := "/" + object.ContainerName + "/" + object.relativePath
Expand Down
Loading