Skip to content

Commit

Permalink
Merge remote-tracking branch 'origin/dev' into adreed/stgexp-wildcard…
Browse files Browse the repository at this point in the history 
…-bug
  • Loading branch information
@adreed-msft
adreed-msft committed Mar 9, 2023
2 parents 03b1b6e + 2fb479f commit 667b948
Show file tree
Hide file tree
Showing 40 changed files with 1,071 additions and 193 deletions.
4 changes: 2 additions & 2 deletions cmd/copy.go
View file
Expand Up @@ -956,8 +956,8 @@ func areBothLocationsSMBAware(fromTo common.FromTo) bool {
func areBothLocationsPOSIXAware(fromTo common.FromTo) bool {
// POSIX properties are stored in blob metadata-- They don't need a special persistence strategy for BlobBlob.
return runtime.GOOS == "linux" && (
// fromTo == common.EFromTo.BlobLocal() || TODO
fromTo == common.EFromTo.LocalBlob()) ||
fromTo == common.EFromTo.BlobLocal() ||
fromTo == common.EFromTo.LocalBlob()) ||
fromTo == common.EFromTo.BlobBlob()
}

Expand Down
16 changes: 8 additions & 8 deletions cmd/copyEnumeratorInit.go
View file
Expand Up @@ -71,8 +71,8 @@ func (cca *CookedCopyCmdArgs) initEnumerator(jobPartOrder common.CopyJobPartOrde
traverser, err = InitResourceTraverser(cca.Source, cca.FromTo.From(), &ctx, &srcCredInfo,
cca.SymlinkHandling, cca.ListOfFilesChannel, cca.Recursive, getRemoteProperties,
cca.IncludeDirectoryStubs, cca.permanentDeleteOption, func(common.EntityType) {}, cca.ListOfVersionIDs,
cca.S2sPreserveBlobTags, common.ESyncHashType.None(), azcopyLogVerbosity.ToPipelineLogLevel(),
cca.CpkOptions, nil /* errorChannel */, cca.StripTopDir)
cca.S2sPreserveBlobTags, common.ESyncHashType.None(), cca.preservePermissions, azcopyLogVerbosity.ToPipelineLogLevel(),
cca.CpkOptions, nil /* errorChannel */, cca.StripTopDir)

if err != nil {
return nil, err
Expand Down Expand Up @@ -118,8 +118,8 @@ func (cca *CookedCopyCmdArgs) initEnumerator(jobPartOrder common.CopyJobPartOrde
return nil, errors.New("cannot use --as-subdir=false with a service level destination")
}

// When copying a container directly to a container, strip the top directory
if srcLevel == ELocationLevel.Container() && dstLevel == ELocationLevel.Container() && cca.FromTo.From().IsRemote() && cca.FromTo.To().IsRemote() {
// When copying a container directly to a container, strip the top directory, unless we're attempting to persist permissions.
if srcLevel == ELocationLevel.Container() && dstLevel == ELocationLevel.Container() && cca.FromTo.From().IsRemote() && cca.FromTo.To().IsRemote() && !cca.preservePermissions.IsTruthy() {
cca.StripTopDir = true
}

Expand Down Expand Up @@ -224,7 +224,7 @@ func (cca *CookedCopyCmdArgs) initEnumerator(jobPartOrder common.CopyJobPartOrde

// decide our folder transfer strategy
var message string
jobPartOrder.Fpo, message = newFolderPropertyOption(cca.FromTo, cca.Recursive, cca.StripTopDir, filters, cca.preserveSMBInfo, cca.preservePermissions.IsTruthy(), cca.preservePOSIXProperties, cca.isHNStoHNS, strings.EqualFold(cca.Destination.Value, common.Dev_Null), cca.IncludeDirectoryStubs)
jobPartOrder.Fpo, message = NewFolderPropertyOption(cca.FromTo, cca.Recursive, cca.StripTopDir, filters, cca.preserveSMBInfo, cca.preservePermissions.IsTruthy(), cca.preservePOSIXProperties, cca.isHNStoHNS, strings.EqualFold(cca.Destination.Value, common.Dev_Null), cca.IncludeDirectoryStubs)
if !cca.dryrunMode {
glcm.Info(message)
}
Expand Down Expand Up @@ -339,8 +339,8 @@ func (cca *CookedCopyCmdArgs) isDestDirectory(dst common.ResourceString, ctx *co

rt, err := InitResourceTraverser(dst, cca.FromTo.To(), ctx, &dstCredInfo, common.ESymlinkHandlingType.Skip(),
nil, false, false, false, common.EPermanentDeleteOption.None(),
func(common.EntityType) {}, cca.ListOfVersionIDs, false, common.ESyncHashType.None(), pipeline.LogNone,
cca.CpkOptions, nil /* errorChannel */, cca.StripTopDir)
func(common.EntityType) {}, cca.ListOfVersionIDs, false, common.ESyncHashType.None(), cca.preservePermissions, pipeline.LogNone,
cca.CpkOptions, nil /* errorChannel */, cca.StripTopDir)

if err != nil {
return false
Expand Down Expand Up @@ -658,7 +658,7 @@ func (cca *CookedCopyCmdArgs) MakeEscapedRelativePath(source bool, dstIsDir bool
}

// we assume that preserveSmbPermissions and preserveSmbInfo have already been validated, such that they are only true if both resource types support them
func newFolderPropertyOption(fromTo common.FromTo, recursive, stripTopDir bool, filters []ObjectFilter, preserveSmbInfo, preserveSmbPermissions, preservePosixProperties, isDfsDfs, isDstNull, includeDirectoryStubs bool) (common.FolderPropertyOption, string) {
func NewFolderPropertyOption(fromTo common.FromTo, recursive, stripTopDir bool, filters []ObjectFilter, preserveSmbInfo, preserveSmbPermissions, preservePosixProperties, isDfsDfs, isDstNull, includeDirectoryStubs bool) (common.FolderPropertyOption, string) {

getSuffix := func(willProcess bool) string {
willProcessString := common.IffString(willProcess, "will be processed", "will not be processed")
Expand Down
10 changes: 5 additions & 5 deletions cmd/copyEnumeratorInit_test.go
View file
Expand Up @@ -49,7 +49,7 @@ func (ce *copyEnumeratorSuite) TestValidateSourceDirThatExists(c *chk.C) {

// List
rawBlobURLWithSAS := scenarioHelper{}.getRawBlobURLWithSAS(c, containerName, dirName)
blobTraverser := newBlobTraverser(&rawBlobURLWithSAS, p, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false)
blobTraverser := newBlobTraverser(&rawBlobURLWithSAS, p, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None())

// dir but recursive flag not set - fail
cca := CookedCopyCmdArgs{StripTopDir: false, Recursive: false}
Expand Down Expand Up @@ -78,7 +78,7 @@ func (ce *copyEnumeratorSuite) TestValidateSourceDirDoesNotExist(c *chk.C) {

// List
rawBlobURLWithSAS := scenarioHelper{}.getRawBlobURLWithSAS(c, containerName, dirName)
blobTraverser := newBlobTraverser(&rawBlobURLWithSAS, p, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false)
blobTraverser := newBlobTraverser(&rawBlobURLWithSAS, p, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None())

// dir but recursive flag not set - fail
cca := CookedCopyCmdArgs{StripTopDir: false, Recursive: false}
Expand Down Expand Up @@ -108,7 +108,7 @@ func (ce *copyEnumeratorSuite) TestValidateSourceFileExists(c *chk.C) {

// List
rawBlobURLWithSAS := scenarioHelper{}.getRawBlobURLWithSAS(c, containerName, fileName)
blobTraverser := newBlobTraverser(&rawBlobURLWithSAS, p, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false)
blobTraverser := newBlobTraverser(&rawBlobURLWithSAS, p, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None())

cca := CookedCopyCmdArgs{StripTopDir: false, Recursive: false}
err := cca.validateSourceDir(blobTraverser)
Expand All @@ -131,7 +131,7 @@ func (ce *copyEnumeratorSuite) TestValidateSourceFileDoesNotExist(c *chk.C) {

// List
rawBlobURLWithSAS := scenarioHelper{}.getRawBlobURLWithSAS(c, containerName, fileName)
blobTraverser := newBlobTraverser(&rawBlobURLWithSAS, p, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false)
blobTraverser := newBlobTraverser(&rawBlobURLWithSAS, p, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None())

cca := CookedCopyCmdArgs{StripTopDir: false, Recursive: false}
err := cca.validateSourceDir(blobTraverser)
Expand All @@ -154,7 +154,7 @@ func (ce *copyEnumeratorSuite) TestValidateSourceWithWildCard(c *chk.C) {

// List
rawBlobURLWithSAS := scenarioHelper{}.getRawBlobURLWithSAS(c, containerName, dirName)
blobTraverser := newBlobTraverser(&rawBlobURLWithSAS, p, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false)
blobTraverser := newBlobTraverser(&rawBlobURLWithSAS, p, ctx, true, true, func(common.EntityType) {}, false, common.CpkOptions{}, false, false, false, common.EPreservePermissionsOption.None())

// dir but recursive flag not set - fail
cca := CookedCopyCmdArgs{StripTopDir: true, Recursive: false}
Expand Down
4 changes: 2 additions & 2 deletions cmd/list.go
View file
Expand Up @@ -224,8 +224,8 @@ func (cooked cookedListCmdArgs) HandleListContainerCommand() (err error) {

traverser, err := InitResourceTraverser(source, cooked.location, &ctx, &credentialInfo, common.ESymlinkHandlingType.Skip(), nil,
true, false, false, common.EPermanentDeleteOption.None(), func(common.EntityType) {},
nil, false, common.ESyncHashType.None(), pipeline.LogNone,
common.CpkOptions{}, nil /* errorChannel */, false)
nil, false, common.ESyncHashType.None(), common.EPreservePermissionsOption.None(),
pipeline.LogNone, common.CpkOptions{}, nil /* errorChannel */, false)

if err != nil {
return fmt.Errorf("failed to initialize traverser: %s", err.Error())
Expand Down
5 changes: 3 additions & 2 deletions cmd/removeEnumerator.go
View file
Expand Up @@ -51,7 +51,8 @@ func newRemoveEnumerator(cca *CookedCopyCmdArgs) (enumerator *CopyEnumerator, er
sourceTraverser, err = InitResourceTraverser(cca.Source, cca.FromTo.From(), &ctx, &cca.credentialInfo,
common.ESymlinkHandlingType.Skip(), cca.ListOfFilesChannel, cca.Recursive, false, cca.IncludeDirectoryStubs,
cca.permanentDeleteOption, func(common.EntityType) {}, cca.ListOfVersionIDs, false,
common.ESyncHashType.None(), azcopyLogVerbosity.ToPipelineLogLevel(), cca.CpkOptions, nil /* errorChannel */, cca.StripTopDir)
common.ESyncHashType.None(), common.EPreservePermissionsOption.None(), azcopyLogVerbosity.ToPipelineLogLevel(),
cca.CpkOptions, nil /* errorChannel */, cca.StripTopDir)

// report failure to create traverser
if err != nil {
Expand Down Expand Up @@ -79,7 +80,7 @@ func newRemoveEnumerator(cca *CookedCopyCmdArgs) (enumerator *CopyEnumerator, er
// (Must enumerate folders when deleting from a folder-aware location. Can't do folder deletion just based on file
// deletion, because that would not handle folders that were empty at the start of the job).
// isHNStoHNS is IGNORED here, because BlobFS locations don't take this route currently.
fpo, message := newFolderPropertyOption(cca.FromTo, cca.Recursive, cca.StripTopDir, filters, false, false, false, false, false, cca.IncludeDirectoryStubs)
fpo, message := NewFolderPropertyOption(cca.FromTo, cca.Recursive, cca.StripTopDir, filters, false, false, false, false, false, cca.IncludeDirectoryStubs)
// do not print Info message if in dry run mode
if !cca.dryrunMode {
glcm.Info(message)
Expand Down
5 changes: 3 additions & 2 deletions cmd/setPropertiesEnumerator.go
View file
Expand Up @@ -51,7 +51,8 @@ func setPropertiesEnumerator(cca *CookedCopyCmdArgs) (enumerator *CopyEnumerator
common.ESymlinkHandlingType.Preserve(), // preserve because we want to index all blobs, including symlink blobs
cca.ListOfFilesChannel, cca.Recursive, false, cca.IncludeDirectoryStubs,
cca.permanentDeleteOption, func(common.EntityType) {}, cca.ListOfVersionIDs, false,
common.ESyncHashType.None(), azcopyLogVerbosity.ToPipelineLogLevel(), cca.CpkOptions, nil /* errorChannel */, cca.StripTopDir)
common.ESyncHashType.None(), common.EPreservePermissionsOption.None(), azcopyLogVerbosity.ToPipelineLogLevel(),
cca.CpkOptions, nil /* errorChannel */, cca.StripTopDir)

// report failure to create traverser
if err != nil {
Expand All @@ -68,7 +69,7 @@ func setPropertiesEnumerator(cca *CookedCopyCmdArgs) (enumerator *CopyEnumerator
filters = append(filters, excludePathFilters...)
filters = append(filters, includeSoftDelete...)

fpo, message := newFolderPropertyOption(cca.FromTo, cca.Recursive, cca.StripTopDir, filters, false, false, false, cca.isHNStoHNS, strings.EqualFold(cca.Destination.Value, common.Dev_Null), cca.IncludeDirectoryStubs)
fpo, message := NewFolderPropertyOption(cca.FromTo, cca.Recursive, cca.StripTopDir, filters, false, false, false, cca.isHNStoHNS, strings.EqualFold(cca.Destination.Value, common.Dev_Null), cca.IncludeDirectoryStubs)
// do not print Info message if in dry run mode
if !cca.dryrunMode {
glcm.Info(message)
Expand Down
10 changes: 5 additions & 5 deletions cmd/syncEnumerator.go
View file
Expand Up @@ -65,8 +65,8 @@ func (cca *cookedSyncCmdArgs) initEnumerator(ctx context.Context) (enumerator *s
if entityType == common.EEntityType.File() {
atomic.AddUint64(&cca.atomicSourceFilesScanned, 1)
}
}, nil, cca.s2sPreserveBlobTags, cca.compareHash, azcopyLogVerbosity.ToPipelineLogLevel(),
cca.cpkOptions, nil /* errorChannel */, false)
}, nil, cca.s2sPreserveBlobTags, cca.compareHash, cca.preservePermissions, azcopyLogVerbosity.ToPipelineLogLevel(),
cca.cpkOptions, nil /* errorChannel */, false)

if err != nil {
return nil, err
Expand All @@ -87,8 +87,8 @@ func (cca *cookedSyncCmdArgs) initEnumerator(ctx context.Context) (enumerator *s
if entityType == common.EEntityType.File() {
atomic.AddUint64(&cca.atomicDestinationFilesScanned, 1)
}
}, nil, cca.s2sPreserveBlobTags, cca.compareHash, azcopyLogVerbosity.ToPipelineLogLevel(),
cca.cpkOptions, nil, false)
}, nil, cca.s2sPreserveBlobTags, cca.compareHash, cca.preservePermissions, azcopyLogVerbosity.ToPipelineLogLevel(),
cca.cpkOptions, nil /* errorChannel */, false)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -131,7 +131,7 @@ func (cca *cookedSyncCmdArgs) initEnumerator(ctx context.Context) (enumerator *s
}

// decide our folder transfer strategy
fpo, folderMessage := newFolderPropertyOption(cca.fromTo, cca.recursive, true, filters, cca.preserveSMBInfo, cca.preservePermissions.IsTruthy(), false, cca.isHNSToHNS, strings.EqualFold(cca.destination.Value, common.Dev_Null), false) // sync always acts like stripTopDir=true
fpo, folderMessage := NewFolderPropertyOption(cca.fromTo, cca.recursive, true, filters, cca.preserveSMBInfo, cca.preservePermissions.IsTruthy(), false, cca.isHNSToHNS, strings.EqualFold(cca.destination.Value, common.Dev_Null), false) // sync always acts like stripTopDir=true
if !cca.dryrunMode {
glcm.Info(folderMessage)
}
Expand Down
10 changes: 5 additions & 5 deletions cmd/zc_enumerator.go
View file
Expand Up @@ -332,7 +332,7 @@ type enumerationCounterFunc func(entityType common.EntityType)
func InitResourceTraverser(resource common.ResourceString, location common.Location, ctx *context.Context,
credential *common.CredentialInfo, symlinkHandling common.SymlinkHandlingType, listOfFilesChannel chan string, recursive, getProperties,
includeDirectoryStubs bool, permanentDeleteOption common.PermanentDeleteOption, incrementEnumerationCounter enumerationCounterFunc, listOfVersionIds chan string,
s2sPreserveBlobTags bool, syncHashType common.SyncHashType, logLevel pipeline.LogLevel, cpkOptions common.CpkOptions, errorChannel chan ErrorFileInfo, stripTopDir bool) (ResourceTraverser, error) {
s2sPreserveBlobTags bool, syncHashType common.SyncHashType, preservePermissions common.PreservePermissionsOption, logLevel pipeline.LogLevel, cpkOptions common.CpkOptions, errorChannel chan ErrorFileInfo, stripTopDir bool) (ResourceTraverser, error) {
var output ResourceTraverser
var p *pipeline.Pipeline

Expand Down Expand Up @@ -380,7 +380,7 @@ func InitResourceTraverser(resource common.ResourceString, location common.Locat
}

output = newListTraverser(resource, location, credential, ctx, recursive, symlinkHandling, getProperties,
listOfFilesChannel, includeDirectoryStubs, incrementEnumerationCounter, s2sPreserveBlobTags, logLevel, cpkOptions)
listOfFilesChannel, includeDirectoryStubs, incrementEnumerationCounter, s2sPreserveBlobTags, logLevel, cpkOptions, syncHashType, preservePermissions)
return output, nil
}

Expand Down Expand Up @@ -408,7 +408,7 @@ func InitResourceTraverser(resource common.ResourceString, location common.Locat

baseResource := resource.CloneWithValue(cleanLocalPath(basePath))
output = newListTraverser(baseResource, location, nil, nil, recursive, symlinkHandling, getProperties,
globChan, includeDirectoryStubs, incrementEnumerationCounter, s2sPreserveBlobTags, logLevel, cpkOptions)
globChan, includeDirectoryStubs, incrementEnumerationCounter, s2sPreserveBlobTags, logLevel, cpkOptions, syncHashType, preservePermissions)
} else {
if ctx != nil {
output = newLocalTraverser(*ctx, resource.ValueLocal(), recursive, stripTopDir, symlinkHandling, syncHashType, incrementEnumerationCounter, errorChannel)
Expand Down Expand Up @@ -443,11 +443,11 @@ func InitResourceTraverser(resource common.ResourceString, location common.Locat
return nil, errors.New(accountTraversalInherentlyRecursiveError)
}

output = newBlobAccountTraverser(resourceURL, *p, *ctx, includeDirectoryStubs, incrementEnumerationCounter, s2sPreserveBlobTags, cpkOptions)
output = newBlobAccountTraverser(resourceURL, *p, *ctx, includeDirectoryStubs, incrementEnumerationCounter, s2sPreserveBlobTags, cpkOptions, preservePermissions)
} else if listOfVersionIds != nil {
output = newBlobVersionsTraverser(resourceURL, *p, *ctx, recursive, includeDirectoryStubs, incrementEnumerationCounter, listOfVersionIds, cpkOptions)
} else {
output = newBlobTraverser(resourceURL, *p, *ctx, recursive, includeDirectoryStubs, incrementEnumerationCounter, s2sPreserveBlobTags, cpkOptions, includeDeleted, includeSnapshot, includeVersion)
output = newBlobTraverser(resourceURL, *p, *ctx, recursive, includeDirectoryStubs, incrementEnumerationCounter, s2sPreserveBlobTags, cpkOptions, includeDeleted, includeSnapshot, includeVersion, preservePermissions)
}
case common.ELocation.File():
resourceURL, err := resource.FullURL()
Expand Down

0 comments on commit 667b948

Please sign in to comment.