diff --git a/.github/dependabot.yml b/.github/dependabot.yml index d608a8244..6ec4829f2 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -5,9 +5,14 @@ updates: directory: "/" # Location of package manifests schedule: interval: "weekly" + # Create security fix PRs only + open-pull-requests-limit: 0 + # Dependencies listed in .github/workflows/*.yml - package-ecosystem: "github-actions" directory: "/" schedule: interval: "weekly" + # Create security fix PRs only + open-pull-requests-limit: 0 diff --git a/CHANGELOG.md b/CHANGELOG.md index 0c0172622..0716130f6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,13 +9,16 @@ - Added `--show-fullpath` flag to `ls`. ([#596](https://github.com/peak/s5cmd/issues/596)) - Added `pipe` command. ([#182](https://github.com/peak/s5cmd/issues/182)) - Added `--show-progress` flag to `cp` to show a progress bar. ([#51](https://github.com/peak/s5cmd/issues/51)) +- Added `--include` flag to `cp`, `rm` and `sync` commands. ([#516](https://github.com/peak/s5cmd/issues/516)) #### Improvements - Implemented concurrent multipart download support for `cat`. ([#245](https://github.com/peak/s5cmd/issues/245)) - Upgraded minimum required Go version to 1.19. ([#583](https://github.com/peak/s5cmd/pull/583)) #### Bugfixes +- Fixed a bug that causes `sync` command with whitespaced flag value to fail. ([#541](https://github.com/peak/s5cmd/issues/541)) - Fixed a bug introduced with `external sort` support in `sync` command which prevents `sync` to an empty destination with `--delete` option. ([#576](https://github.com/peak/s5cmd/issues/576)) +- Fixed a bug in `sync` command, which previously caused the command to continue running even if an error was received from the destination bucket. ([#564](https://github.com/peak/s5cmd/issues/564)) - Fixed a bug that causes local files to be lost if downloads fail. ([#479](https://github.com/peak/s5cmd/issues/479)) ## v2.1.0 - 19 Jun 2023 diff --git a/README.md b/README.md index 726f57c5e..aceeb551c 100644 --- a/README.md +++ b/README.md @@ -301,6 +301,29 @@ folder hierarchy. ⚠️ Copying objects (from S3 to S3) larger than 5GB is not supported yet. We have an [open ticket](https://github.com/peak/s5cmd/issues/29) to track the issue. +#### Using Exclude and Include Filters +`s5cmd` supports the `--exclude` and `--include` flags, which can be used to specify patterns for objects to be excluded or included in commands. + +- The `--exclude` flag specifies objects that should be excluded from the operation. Any object that matches the pattern will be skipped. +- The `--include` flag specifies objects that should be included in the operation. Only objects that match the pattern will be handled. +- If both flags are used, `--exclude` has precedence over `--include`. This means that if an object URL matches any of the `--exclude` patterns, the object will be skipped, even if it also matches one of the `--include` patterns. +- The order of the flags does not affect the results (unlike `aws-cli`). + +The command below will delete only objects that end with `.log`. + + s5cmd rm --include "*.log" 's3://bucket/logs/2020/*' + +The command below will delete all objects except those that end with `.log` or `.txt`. + + s5cmd rm --exclude "*.log" --exclude "*.txt" 's3://bucket/logs/2020/*' + +If you wish, you can use multiple flags, like below. It will download objects that start with `request` and end with `.log`. + + s5cmd cp --include "*.log" --include "request*" 's3://bucket/logs/2020/*' . + +Using a combination of `--include` and `--exclude` also possible. The command below will only sync objects that end with `.log` and `.txt` but exclude those that start with `access_`. For example, `request.log`, and `license.txt` will be included, while `access_log.txt`, and `readme.md` are excluded. + + s5cmd sync --include "*log" --exclude "access_*" --include "*txt" 's3://bucket/logs/*' . #### Select JSON object content using SQL `s5cmd` supports the `SelectObjectContent` S3 operation, and will run your diff --git a/command/context.go b/command/context.go index 3e3f5d08d..1eb78fb3d 100644 --- a/command/context.go +++ b/command/context.go @@ -73,7 +73,7 @@ func generateCommand(c *cli.Context, cmd string, defaultFlags map[string]interfa flags := []string{} for flagname, flagvalue := range defaultFlags { - flags = append(flags, fmt.Sprintf("--%s=%v", flagname, flagvalue)) + flags = append(flags, fmt.Sprintf("--%s='%v'", flagname, flagvalue)) } isDefaultFlag := func(flagname string) bool { @@ -88,7 +88,7 @@ func generateCommand(c *cli.Context, cmd string, defaultFlags map[string]interfa } for _, flagvalue := range contextValue(c, flagname) { - flags = append(flags, fmt.Sprintf("--%s=%s", flagname, flagvalue)) + flags = append(flags, fmt.Sprintf("--%s='%s'", flagname, flagvalue)) } } diff --git a/command/context_test.go b/command/context_test.go index 3e7536319..e7783356f 100644 --- a/command/context_test.go +++ b/command/context_test.go @@ -2,7 +2,6 @@ package command import ( "flag" - "strings" "testing" "github.com/google/go-cmp/cmp" @@ -46,7 +45,25 @@ func TestGenerateCommand(t *testing.T) { mustNewURL(t, "s3://bucket/key1"), mustNewURL(t, "s3://bucket/key2"), }, - expectedCommand: `cp --acl=public-read --raw=true "s3://bucket/key1" "s3://bucket/key2"`, + expectedCommand: `cp --acl='public-read' --raw='true' "s3://bucket/key1" "s3://bucket/key2"`, + }, + { + name: "cli-flag-with-whitespaced-flag-value", + cmd: "cp", + flags: []cli.Flag{ + &cli.StringFlag{ + Name: "cache-control", + Value: "public, max-age=31536000, immutable", + }, + }, + defaultFlags: map[string]interface{}{ + "raw": true, + }, + urls: []*url.URL{ + mustNewURL(t, "s3://bucket/key1"), + mustNewURL(t, "s3://bucket/key2"), + }, + expectedCommand: `cp --cache-control='public, max-age=31536000, immutable' --raw='true' "s3://bucket/key1" "s3://bucket/key2"`, }, { name: "same-flag-should-be-ignored-if-given-from-both-default-and-cli-flags", @@ -64,7 +81,7 @@ func TestGenerateCommand(t *testing.T) { mustNewURL(t, "s3://bucket/key1"), mustNewURL(t, "s3://bucket/key2"), }, - expectedCommand: `cp --raw=true "s3://bucket/key1" "s3://bucket/key2"`, + expectedCommand: `cp --raw='true' "s3://bucket/key1" "s3://bucket/key2"`, }, { name: "ignore-non-shared-flag", @@ -101,7 +118,7 @@ func TestGenerateCommand(t *testing.T) { mustNewURL(t, "s3://bucket/key1"), mustNewURL(t, "s3://bucket/key2"), }, - expectedCommand: `cp --concurrency=6 --flatten=true --force-glacier-transfer=true --raw=true "s3://bucket/key1" "s3://bucket/key2"`, + expectedCommand: `cp --concurrency='6' --flatten='true' --force-glacier-transfer='true' --raw='true' "s3://bucket/key1" "s3://bucket/key2"`, }, { name: "string-slice-flag", @@ -116,7 +133,7 @@ func TestGenerateCommand(t *testing.T) { mustNewURL(t, "/source/dir"), mustNewURL(t, "s3://bucket/prefix/"), }, - expectedCommand: `cp --exclude=*.log --exclude=*.txt "/source/dir" "s3://bucket/prefix/"`, + expectedCommand: `cp --exclude='*.log' --exclude='*.txt' "/source/dir" "s3://bucket/prefix/"`, }, { name: "command-with-multiple-args", @@ -155,10 +172,12 @@ func TestGenerateCommand(t *testing.T) { // and methods to update context are package-private, so write simple // flag parser to update context value. set.VisitAll(func(f *flag.Flag) { - value := strings.Trim(f.Value.String(), "[") - value = strings.Trim(value, "]") - for _, v := range strings.Fields(value) { - ctx.Set(f.Name, v) + if v, ok := f.Value.(*cli.StringSlice); ok { + for _, s := range v.Value() { + ctx.Set(f.Name, s) + } + } else { + ctx.Set(f.Name, f.Value.String()) } }) diff --git a/command/cp.go b/command/cp.go index 1c0ea75fa..36f71a3fe 100644 --- a/command/cp.go +++ b/command/cp.go @@ -9,6 +9,7 @@ import ( "net/http" "os" "path/filepath" + "regexp" "strings" "sync" @@ -97,13 +98,16 @@ Examples: 19. Copy all files from S3 bucket to another S3 bucket but exclude the ones starts with log > s5cmd {{.HelpName}} --exclude "log*" "s3://bucket/*" s3://destbucket - 20. Download an S3 object from a requester pays bucket + 20. Copy all files from S3 bucket to another S3 bucket but only the ones starts with log + > s5cmd {{.HelpName}} --include "log*" "s3://bucket/*" s3://destbucket + + 21. Download an S3 object from a requester pays bucket > s5cmd --request-payer=requester {{.HelpName}} s3://bucket/prefix/object.gz . - 21. Upload a file to S3 with a content-type and content-encoding header + 22. Upload a file to S3 with a content-type and content-encoding header > s5cmd --content-type "text/css" --content-encoding "br" myfile.css.br s3://bucket/ - 22. Download the specific version of a remote object to working directory + 23. Download the specific version of a remote object to working directory > s5cmd {{.HelpName}} --version-id VERSION_ID s3://bucket/prefix/object . ` @@ -169,6 +173,10 @@ func NewSharedFlags() []cli.Flag { Name: "exclude", Usage: "exclude objects with given pattern", }, + &cli.StringSliceFlag{ + Name: "include", + Usage: "include objects with given pattern", + }, &cli.BoolFlag{ Name: "raw", Usage: "disable the wildcard operations, useful with filenames that contains glob characters", @@ -282,6 +290,7 @@ type Copy struct { forceGlacierTransfer bool ignoreGlacierWarnings bool exclude []string + include []string cacheControl string expires string contentType string @@ -290,6 +299,10 @@ type Copy struct { showProgress bool progressbar progressbar.ProgressBar + // patterns + excludePatterns []*regexp.Regexp + includePatterns []*regexp.Regexp + // region settings srcRegion string dstRegion string @@ -346,6 +359,7 @@ func NewCopy(c *cli.Context, deleteSource bool) (*Copy, error) { forceGlacierTransfer: c.Bool("force-glacier-transfer"), ignoreGlacierWarnings: c.Bool("ignore-glacier-warnings"), exclude: c.StringSlice("exclude"), + include: c.StringSlice("include"), cacheControl: c.String("cache-control"), expires: c.String("expires"), contentType: c.String("content-type"), @@ -422,14 +436,27 @@ func (c Copy) Run(ctx context.Context) error { isBatch = obj != nil && obj.Type.IsDir() } - excludePatterns, err := createExcludesFromWildcard(c.exclude) + c.excludePatterns, err = createRegexFromWildcard(c.exclude) + if err != nil { + printError(c.fullCommand, c.op, err) + return err + } + + c.includePatterns, err = createRegexFromWildcard(c.include) if err != nil { printError(c.fullCommand, c.op, err) return err } for object := range objch { - if object.Type.IsDir() || errorpkg.IsCancelation(object.Err) { + if errorpkg.IsCancelation(object.Err) || object.Type.IsDir() { + continue + } + + if !object.Type.IsRegular() { + err := fmt.Errorf("object '%v' is not a regular file", object) + merrorObjects = multierror.Append(merrorObjects, err) + printError(c.fullCommand, c.op, err) continue } @@ -448,7 +475,11 @@ func (c Copy) Run(ctx context.Context) error { continue } - if isURLExcluded(excludePatterns, object.URL.Path, c.src.Prefix) { + isExcluded, err := isObjectExcluded(object, c.excludePatterns, c.includePatterns, c.src.Prefix) + if err != nil { + printError(c.fullCommand, c.op, err) + } + if isExcluded { continue } diff --git a/command/du.go b/command/du.go index 3a46aab03..8812c9582 100644 --- a/command/du.go +++ b/command/du.go @@ -144,7 +144,7 @@ func (sz Size) Run(ctx context.Context) error { var merror error - excludePatterns, err := createExcludesFromWildcard(sz.exclude) + excludePatterns, err := createRegexFromWildcard(sz.exclude) if err != nil { printError(sz.fullCommand, sz.op, err) return err @@ -161,7 +161,7 @@ func (sz Size) Run(ctx context.Context) error { continue } - if isURLExcluded(excludePatterns, object.URL.Path, sz.src.Prefix) { + if isURLMatched(excludePatterns, object.URL.Path, sz.src.Prefix) { continue } diff --git a/command/exclude.go b/command/exclude.go deleted file mode 100644 index 36d9a9aa9..000000000 --- a/command/exclude.go +++ /dev/null @@ -1,44 +0,0 @@ -package command - -import ( - "path/filepath" - "regexp" - "strings" - - "github.com/peak/s5cmd/v2/strutil" -) - -// createExcludesFromWildcard creates regex strings from wildcard. -func createExcludesFromWildcard(inputExcludes []string) ([]*regexp.Regexp, error) { - var result []*regexp.Regexp - for _, input := range inputExcludes { - if input != "" { - regex := strutil.WildCardToRegexp(input) - regex = strutil.MatchFromStartToEnd(regex) - regex = strutil.AddNewLineFlag(regex) - regexpCompiled, err := regexp.Compile(regex) - if err != nil { - return nil, err - } - result = append(result, regexpCompiled) - } - } - return result, nil -} - -// isURLExcluded checks whether given urlPath matches any of the exclude patterns. -func isURLExcluded(excludePatterns []*regexp.Regexp, urlPath, sourcePrefix string) bool { - if len(excludePatterns) == 0 { - return false - } - if !strings.HasSuffix(sourcePrefix, "/") { - sourcePrefix += "/" - } - sourcePrefix = filepath.ToSlash(sourcePrefix) - for _, excludePattern := range excludePatterns { - if excludePattern.MatchString(strings.TrimPrefix(urlPath, sourcePrefix)) { - return true - } - } - return false -} diff --git a/command/expand.go b/command/expand.go index 1e94fd16a..d079c2251 100644 --- a/command/expand.go +++ b/command/expand.go @@ -19,7 +19,7 @@ func expandSource( followSymlinks bool, srcurl *url.URL, ) (<-chan *storage.Object, error) { - var isDir bool + var objType storage.ObjectType // if the source is local, we send a Stat call to know if we have // directory or file to walk. For remote storage, we don't want to send // Stat since it doesn't have any folder semantics. @@ -28,17 +28,17 @@ func expandSource( if err != nil { return nil, err } - isDir = obj.Type.IsDir() + objType = obj.Type } // call storage.List for only walking operations. - if srcurl.IsWildcard() || srcurl.AllVersions || isDir { + if srcurl.IsWildcard() || srcurl.AllVersions || objType.IsDir() { return client.List(ctx, srcurl, followSymlinks), nil } ch := make(chan *storage.Object, 1) if storage.ShouldProcessURL(srcurl, followSymlinks) { - ch <- &storage.Object{URL: srcurl} + ch <- &storage.Object{URL: srcurl, Type: objType} } close(ch) return ch, nil diff --git a/command/ls.go b/command/ls.go index 2fe591d7a..87ff1e06b 100644 --- a/command/ls.go +++ b/command/ls.go @@ -188,7 +188,7 @@ func (l List) Run(ctx context.Context) error { var merror error - excludePatterns, err := createExcludesFromWildcard(l.exclude) + excludePatterns, err := createRegexFromWildcard(l.exclude) if err != nil { printError(l.fullCommand, l.op, err) return err @@ -205,7 +205,7 @@ func (l List) Run(ctx context.Context) error { continue } - if isURLExcluded(excludePatterns, object.URL.Path, l.src.Prefix) { + if isURLMatched(excludePatterns, object.URL.Path, l.src.Prefix) { continue } diff --git a/command/rm.go b/command/rm.go index 94698abf0..3ba092b23 100644 --- a/command/rm.go +++ b/command/rm.go @@ -3,6 +3,7 @@ package command import ( "context" "fmt" + "regexp" "github.com/hashicorp/go-multierror" "github.com/urfave/cli/v2" @@ -38,17 +39,20 @@ Examples: 5. Delete all matching objects but exclude the ones with .txt extension or starts with "main" > s5cmd {{.HelpName}} --exclude "*.txt" --exclude "main*" "s3://bucketname/prefix/*" + + 6. Delete all matching objects but only the ones with .txt extension or starts with "main" + > s5cmd {{.HelpName}} --include "*.txt" --include "main*" "s3://bucketname/prefix/*" - 6. Delete the specific version of a remote object's content to stdout + 7. Delete the specific version of a remote object's content to stdout > s5cmd {{.HelpName}} --version-id VERSION_ID s3://bucket/prefix/object - 7. Delete all versions of an object in the bucket + 8. Delete all versions of an object in the bucket > s5cmd {{.HelpName}} --all-versions s3://bucket/object - 8. Delete all versions of all objects that starts with a prefix in the bucket + 9. Delete all versions of all objects that starts with a prefix in the bucket > s5cmd {{.HelpName}} --all-versions "s3://bucket/prefix*" - 9. Delete all versions of all objects in the bucket + 10. Delete all versions of all objects in the bucket > s5cmd {{.HelpName}} --all-versions "s3://bucket/*" ` @@ -66,6 +70,10 @@ func NewDeleteCommand() *cli.Command { Name: "exclude", Usage: "exclude objects with given pattern", }, + &cli.StringSliceFlag{ + Name: "include", + Usage: "include objects with given pattern", + }, &cli.BoolFlag{ Name: "all-versions", Usage: "list all versions of object(s)", @@ -94,6 +102,18 @@ func NewDeleteCommand() *cli.Command { return err } + excludePatterns, err := createRegexFromWildcard(c.StringSlice("exclude")) + if err != nil { + printError(fullCommand, c.Command.Name, err) + return err + } + + includePatterns, err := createRegexFromWildcard(c.StringSlice("include")) + if err != nil { + printError(fullCommand, c.Command.Name, err) + return err + } + return Delete{ src: srcUrls, op: c.Command.Name, @@ -101,6 +121,11 @@ func NewDeleteCommand() *cli.Command { // flags exclude: c.StringSlice("exclude"), + include: c.StringSlice("include"), + + // patterns + excludePatterns: excludePatterns, + includePatterns: includePatterns, storageOpts: NewStorageOpts(c), }.Run(c.Context) @@ -119,6 +144,11 @@ type Delete struct { // flag options exclude []string + include []string + + // patterns + excludePatterns []*regexp.Regexp + includePatterns []*regexp.Regexp // storage options storageOpts storage.Options @@ -135,12 +165,6 @@ func (d Delete) Run(ctx context.Context) error { return err } - excludePatterns, err := createExcludesFromWildcard(d.exclude) - if err != nil { - printError(d.fullCommand, d.op, err) - return err - } - objch := expandSources(ctx, client, false, d.src...) var ( @@ -164,7 +188,11 @@ func (d Delete) Run(ctx context.Context) error { continue } - if isURLExcluded(excludePatterns, object.URL.Path, srcurl.Prefix) { + isExcluded, err := isObjectExcluded(object, d.excludePatterns, d.includePatterns, srcurl.Prefix) + if err != nil { + printError(d.fullCommand, d.op, err) + } + if isExcluded { continue } diff --git a/command/run.go b/command/run.go index f6ac6687f..5ef3c7890 100644 --- a/command/run.go +++ b/command/run.go @@ -3,6 +3,7 @@ package command import ( "bufio" "context" + "errors" "flag" "fmt" "io" @@ -197,6 +198,9 @@ func (r *Reader) read() { } if err != nil { if err == io.EOF { + if errors.Is(r.ctx.Err(), context.Canceled) { + r.err = r.ctx.Err() + } return } r.err = multierror.Append(r.err, err) diff --git a/command/select.go b/command/select.go index eb2c2eddf..cbd42990d 100644 --- a/command/select.go +++ b/command/select.go @@ -191,7 +191,7 @@ func (s Select) Run(ctx context.Context) error { } }() - excludePatterns, err := createExcludesFromWildcard(s.exclude) + excludePatterns, err := createRegexFromWildcard(s.exclude) if err != nil { printError(s.fullCommand, s.op, err) return err @@ -217,7 +217,7 @@ func (s Select) Run(ctx context.Context) error { continue } - if isURLExcluded(excludePatterns, object.URL.Path, s.src.Prefix) { + if isURLMatched(excludePatterns, object.URL.Path, s.src.Prefix) { continue } diff --git a/command/sync.go b/command/sync.go index 39f497f6f..c848e7abf 100644 --- a/command/sync.go +++ b/command/sync.go @@ -9,11 +9,13 @@ import ( "strings" "sync" + "github.com/aws/aws-sdk-go/aws/awserr" "github.com/hashicorp/go-multierror" "github.com/lanrat/extsort" "github.com/urfave/cli/v2" errorpkg "github.com/peak/s5cmd/v2/error" + "github.com/peak/s5cmd/v2/log" "github.com/peak/s5cmd/v2/log/stat" "github.com/peak/s5cmd/v2/parallel" "github.com/peak/s5cmd/v2/storage" @@ -64,6 +66,9 @@ Examples: 10. Sync all files to S3 bucket but exclude the ones with txt and gz extension > s5cmd {{.HelpName}} --exclude "*.txt" --exclude "*.gz" dir/ s3://bucket + + 11. Sync all files to S3 bucket but include the only ones with txt and gz extension + > s5cmd {{.HelpName}} --include "*.txt" --include "*.gz" dir/ s3://bucket ` func NewSyncCommandFlags() []cli.Flag { @@ -76,6 +81,10 @@ func NewSyncCommandFlags() []cli.Flag { Name: "size-only", Usage: "make size of object only criteria to decide whether an object should be synced", }, + &cli.BoolFlag{ + Name: "exit-on-error", + Usage: "stops the sync process if an error is received", + }, } sharedFlags := NewSharedFlags() return append(syncFlags, sharedFlags...) @@ -119,8 +128,9 @@ type Sync struct { fullCommand string // flags - delete bool - sizeOnly bool + delete bool + sizeOnly bool + exitOnError bool // s3 options storageOpts storage.Options @@ -142,8 +152,9 @@ func NewSync(c *cli.Context) Sync { fullCommand: commandFromContext(c), // flags - delete: c.Bool("delete"), - sizeOnly: c.Bool("size-only"), + delete: c.Bool("delete"), + sizeOnly: c.Bool("size-only"), + exitOnError: c.Bool("exit-on-error"), // flags followSymlinks: !c.Bool("no-follow-symlinks"), @@ -169,7 +180,9 @@ func (s Sync) Run(c *cli.Context) error { return err } - sourceObjects, destObjects, err := s.getSourceAndDestinationObjects(c.Context, srcurl, dsturl) + ctx, cancel := context.WithCancel(c.Context) + + sourceObjects, destObjects, err := s.getSourceAndDestinationObjects(ctx, cancel, srcurl, dsturl) if err != nil { printError(s.fullCommand, s.op, err) return err @@ -177,12 +190,12 @@ func (s Sync) Run(c *cli.Context) error { isBatch := srcurl.IsWildcard() if !isBatch && !srcurl.IsRemote() { - sourceClient, err := storage.NewClient(c.Context, srcurl, s.storageOpts) + sourceClient, err := storage.NewClient(ctx, srcurl, s.storageOpts) if err != nil { return err } - obj, err := sourceClient.Stat(c.Context, srcurl) + obj, err := sourceClient.Stat(ctx, srcurl) if err != nil { return err } @@ -221,7 +234,7 @@ func (s Sync) Run(c *cli.Context) error { // Create commands in background. go s.planRun(c, onlySource, onlyDest, commonObjects, dsturl, strategy, pipeWriter, isBatch) - err = NewRun(c, pipeReader).Run(c.Context) + err = NewRun(c, pipeReader).Run(ctx) return multierror.Append(err, merrorWaiter).ErrorOrNil() } @@ -284,7 +297,7 @@ func compareObjects(sourceObjects, destObjects chan *storage.Object) (chan *url. // getSourceAndDestinationObjects returns source and destination objects from // given URLs. The returned channels gives objects sorted in ascending order // with respect to their url.Relative path. See also storage.Less. -func (s Sync) getSourceAndDestinationObjects(ctx context.Context, srcurl, dsturl *url.URL) (chan *storage.Object, chan *storage.Object, error) { +func (s Sync) getSourceAndDestinationObjects(ctx context.Context, cancel context.CancelFunc, srcurl, dsturl *url.URL) (chan *storage.Object, chan *storage.Object, error) { sourceClient, err := storage.NewClient(ctx, srcurl, s.storageOpts) if err != nil { return nil, nil, err @@ -332,6 +345,15 @@ func (s Sync) getSourceAndDestinationObjects(ctx context.Context, srcurl, dsturl defer close(filteredSrcObjectChannel) // filter and redirect objects for st := range unfilteredSrcObjectChannel { + if st.Err != nil && s.shouldStopSync(st.Err) { + msg := log.ErrorMessage{ + Err: cleanupError(st.Err), + Command: s.fullCommand, + Operation: s.op, + } + log.Error(msg) + cancel() + } if s.shouldSkipObject(st, true) { continue } @@ -368,9 +390,17 @@ func (s Sync) getSourceAndDestinationObjects(ctx context.Context, srcurl, dsturl go func() { defer close(filteredDstObjectChannel) - // filter and redirect objects for dt := range unfilteredDestObjectsChannel { + if dt.Err != nil && s.shouldStopSync(dt.Err) { + msg := log.ErrorMessage{ + Err: cleanupError(dt.Err), + Command: s.fullCommand, + Operation: s.op, + } + log.Error(msg) + cancel() + } if s.shouldSkipObject(dt, false) { continue } @@ -538,3 +568,17 @@ func (s Sync) shouldSkipObject(object *storage.Object, verbose bool) bool { } return false } + +// shouldStopSync determines whether a sync process should be stopped or not. +func (s Sync) shouldStopSync(err error) bool { + if err == storage.ErrNoObjectFound { + return false + } + if awsErr, ok := err.(awserr.Error); ok { + switch awsErr.Code() { + case "AccessDenied", "NoSuchBucket": + return true + } + } + return s.exitOnError +} diff --git a/command/wildcard.go b/command/wildcard.go new file mode 100644 index 000000000..73ac239e7 --- /dev/null +++ b/command/wildcard.go @@ -0,0 +1,57 @@ +package command + +import ( + "path/filepath" + "regexp" + "strings" + + "github.com/peak/s5cmd/v2/storage" + "github.com/peak/s5cmd/v2/strutil" +) + +// createRegexFromWildcard creates regex strings from wildcard. +func createRegexFromWildcard(wildcards []string) ([]*regexp.Regexp, error) { + var result []*regexp.Regexp + for _, input := range wildcards { + if input != "" { + regex := strutil.WildCardToRegexp(input) + regex = strutil.MatchFromStartToEnd(regex) + regex = strutil.AddNewLineFlag(regex) + regexpCompiled, err := regexp.Compile(regex) + if err != nil { + return nil, err + } + result = append(result, regexpCompiled) + } + } + return result, nil +} + +func isURLMatched(regexPatterns []*regexp.Regexp, urlPath, sourcePrefix string) bool { + if len(regexPatterns) == 0 { + return false + } + if !strings.HasSuffix(sourcePrefix, "/") { + sourcePrefix += "/" + } + sourcePrefix = filepath.ToSlash(sourcePrefix) + for _, regexPattern := range regexPatterns { + if regexPattern.MatchString(strings.TrimPrefix(urlPath, sourcePrefix)) { + return true + } + } + return false +} + +func isObjectExcluded(object *storage.Object, excludePatterns []*regexp.Regexp, includePatterns []*regexp.Regexp, prefix string) (bool, error) { + if err := object.Err; err != nil { + return true, err + } + if len(excludePatterns) > 0 && isURLMatched(excludePatterns, object.URL.Path, prefix) { + return true, nil + } + if len(includePatterns) > 0 { + return !isURLMatched(includePatterns, object.URL.Path, prefix), nil + } + return false, nil +} diff --git a/command/wildcard_test.go b/command/wildcard_test.go new file mode 100644 index 000000000..4c3a8fae2 --- /dev/null +++ b/command/wildcard_test.go @@ -0,0 +1,86 @@ +package command + +import ( + "testing" + + "github.com/peak/s5cmd/v2/storage" + "github.com/peak/s5cmd/v2/storage/url" + "gotest.tools/v3/assert" +) + +func TestIsObjectExcluded(t *testing.T) { + t.Parallel() + + testcases := []struct { + excludePatterns []string + includePatterns []string + objects []string + filteredObjects []string + }{ + { + excludePatterns: []string{"*.txt", "*.log"}, + includePatterns: []string{"file-*.doc"}, + objects: []string{"document.txt", "file-2.log", "file-1.doc", "image.png"}, + filteredObjects: []string{"file-1.doc"}, + }, + { + excludePatterns: []string{"secret-*"}, + includePatterns: []string{"*.txt", "*.log"}, + objects: []string{"secret-passwords.txt", "file-1.txt", "file-2.txt", "image.png"}, + filteredObjects: []string{"file-1.txt", "file-2.txt"}, + }, + { + excludePatterns: []string{}, + includePatterns: []string{"*.png"}, + objects: []string{"secret-passwords.txt", "file-1.txt", "file-2.txt", "image.png"}, + filteredObjects: []string{"image.png"}, + }, + { + excludePatterns: []string{"file*"}, + includePatterns: []string{}, + objects: []string{"readme.md", "file-1.txt", "file-2.txt", "image.png"}, + filteredObjects: []string{"readme.md", "image.png"}, + }, + { + excludePatterns: []string{"file*"}, + includePatterns: []string{"*txt"}, + objects: []string{"readme.txt", "file-1.txt", "file-2.txt", "license.txt"}, + filteredObjects: []string{"readme.txt", "license.txt"}, + }, + { + excludePatterns: []string{"*tmp", "*.txt"}, + includePatterns: []string{"*png", "*.doc*"}, + objects: []string{"readme.txt", "license.txt", "cache.tmp", "image.png", "eula.doc", "eula.docx", "personaldoc"}, + filteredObjects: []string{"image.png", "eula.doc", "eula.docx"}, + }, + } + + for _, tc := range testcases { + tc := tc + + excludeRegex, err := createRegexFromWildcard(tc.excludePatterns) + if err != nil { + t.Error(err) + } + + includeRegex, err := createRegexFromWildcard(tc.includePatterns) + if err != nil { + t.Error(err) + } + + var filteredObjects []string + + for _, object := range tc.objects { + skip, err := isObjectExcluded(&storage.Object{URL: &url.URL{Path: object}}, excludeRegex, includeRegex, "") + if err != nil { + t.Fatal(err) + } + if skip { + continue + } + filteredObjects = append(filteredObjects, object) + } + + assert.DeepEqual(t, tc.filteredObjects, filteredObjects) + } +} diff --git a/e2e/cp_test.go b/e2e/cp_test.go index b4519b548..92773ede3 100644 --- a/e2e/cp_test.go +++ b/e2e/cp_test.go @@ -24,6 +24,7 @@ package e2e import ( "fmt" + "net" "os" "path/filepath" "runtime" @@ -4196,3 +4197,203 @@ func TestCountingWriter(t *testing.T) { expected := fs.Expected(t, fs.WithFile(filename, content, fs.WithMode(0644))) assert.Assert(t, fs.Equal(cmd.Dir, expected)) } + +// It should skip special files +func TestUploadingSocketFile(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip() + } + + t.Parallel() + + s3client, s5cmd := setup(t) + bucket := s3BucketFromTestName(t) + createBucket(t, s3client, bucket) + + workdir := fs.NewDir(t, t.Name()) + defer workdir.Remove() + + sockaddr := workdir.Join("/s5cmd.sock") + ln, err := net.Listen("unix", sockaddr) + if err != nil { + t.Fatal(err) + } + + t.Cleanup(func() { + ln.Close() + os.Remove(sockaddr) + }) + + cmd := s5cmd("cp", sockaddr, "s3://"+bucket+"/") + result := icmd.RunCmd(cmd, withWorkingDir(workdir)) + + // assert error message + assertLines(t, result.Stderr(), map[int]compareFunc{ + 0: contains(`is not a regular file`), + }) + + // assert logs are empty (no copy) + assertLines(t, result.Stdout(), nil) + + // assert exit code + result.Assert(t, icmd.Expected{ExitCode: 1}) +} + +// cp --include "*.py" s3://bucket/* . +func TestCopyS3ObjectsWithIncludeFilter(t *testing.T) { + t.Parallel() + + s3client, s5cmd := setup(t) + + bucket := s3BucketFromTestName(t) + createBucket(t, s3client, bucket) + + const ( + includePattern = "*.py" + fileContent = "content" + ) + + files := [...]string{ + "file1.py", + "file2.py", + "file.txt", + "a.txt", + "src/file.txt", + } + + for _, filename := range files { + putFile(t, s3client, bucket, filename, fileContent) + } + + srcpath := fmt.Sprintf("s3://%s", bucket) + + cmd := s5cmd("cp", "--include", includePattern, srcpath+"/*", ".") + result := icmd.RunCmd(cmd) + + result.Assert(t, icmd.Success) + + assertLines(t, result.Stdout(), map[int]compareFunc{ + 0: equals("cp %v/file1.py %s", srcpath, files[0]), + 1: equals("cp %v/file2.py %s", srcpath, files[1]), + }, sortInput(true)) + + // assert s3 + for _, f := range files { + assert.Assert(t, ensureS3Object(s3client, bucket, f, fileContent)) + } + + expectedFileSystem := []fs.PathOp{ + fs.WithFile("file1.py", fileContent), + fs.WithFile("file2.py", fileContent), + } + // assert local filesystem + expected := fs.Expected(t, expectedFileSystem...) + assert.Assert(t, fs.Equal(cmd.Dir, expected)) +} + +// cp --include "file*" --exclude "*.py" s3://bucket/* . +func TestCopyS3ObjectsWithIncludeExcludeFilter(t *testing.T) { + t.Parallel() + + s3client, s5cmd := setup(t) + + bucket := s3BucketFromTestName(t) + createBucket(t, s3client, bucket) + + const ( + includePattern = "file*" + excludePattern = "*.py" + fileContent = "content" + ) + + files := [...]string{ + "file1.py", + "file2.py", + "test.py", + "app.py", + "docs/readme.md", + } + + for _, filename := range files { + putFile(t, s3client, bucket, filename, fileContent) + } + + srcpath := fmt.Sprintf("s3://%s", bucket) + + cmd := s5cmd("cp", "--include", includePattern, "--exclude", excludePattern, srcpath+"/*", ".") + result := icmd.RunCmd(cmd) + + result.Assert(t, icmd.Success) + assertLines(t, result.Stdout(), map[int]compareFunc{}, sortInput(true)) + + // assert s3 + for _, f := range files { + assert.Assert(t, ensureS3Object(s3client, bucket, f, fileContent)) + } + + expectedFileSystem := []fs.PathOp{} + // assert local filesystem + expected := fs.Expected(t, expectedFileSystem...) + assert.Assert(t, fs.Equal(cmd.Dir, expected)) +} + +// cp --exclude "file*" --exclude "vendor/*" --include "*.py" --include "*.go" s3://bucket/* . +func TestCopyS3ObjectsWithIncludeExcludeFilter2(t *testing.T) { + t.Parallel() + + s3client, s5cmd := setup(t) + + bucket := s3BucketFromTestName(t) + createBucket(t, s3client, bucket) + + const ( + includePattern = "*.py" + includePattern2 = "*.go" + excludePattern = "file*" + excludePattern2 = "vendor/*" + fileContent = "content" + ) + + files := [...]string{ + "file1.py", + "file2.py", + "file1.go", + "file2.go", + "test.py", + "app.py", + "app.go", + "vendor/package.go", + "docs/readme.md", + } + + for _, filename := range files { + putFile(t, s3client, bucket, filename, fileContent) + } + + srcpath := fmt.Sprintf("s3://%s", bucket) + + cmd := s5cmd("cp", "--exclude", excludePattern, "--exclude", excludePattern2, "--include", includePattern, "--include", includePattern2, srcpath+"/*", ".") + result := icmd.RunCmd(cmd) + + result.Assert(t, icmd.Success) + + assertLines(t, result.Stdout(), map[int]compareFunc{ + 0: equals("cp %v/app.go %s", srcpath, files[6]), + 1: equals("cp %v/app.py %s", srcpath, files[5]), + 2: equals("cp %v/test.py %s", srcpath, files[4]), + }, sortInput(true)) + + // assert s3 + for _, f := range files { + assert.Assert(t, ensureS3Object(s3client, bucket, f, fileContent)) + } + + expectedFileSystem := []fs.PathOp{ + fs.WithFile("test.py", fileContent), + fs.WithFile("app.py", fileContent), + fs.WithFile("app.go", fileContent), + } + // assert local filesystem + expected := fs.Expected(t, expectedFileSystem...) + assert.Assert(t, fs.Equal(cmd.Dir, expected)) +} diff --git a/e2e/rm_test.go b/e2e/rm_test.go index 4a01b6a40..43c6cb628 100644 --- a/e2e/rm_test.go +++ b/e2e/rm_test.go @@ -1301,3 +1301,155 @@ func TestRemoveByVersionID(t *testing.T) { result = icmd.RunCmd(cmd) assert.Assert(t, result.Stdout() == "") } + +// rm --include "*.py" s3://bucket/ +func TestRemoveS3ObjectsWithIncludeFilter(t *testing.T) { + t.Parallel() + + s3client, s5cmd := setup(t) + + bucket := s3BucketFromTestName(t) + createBucket(t, s3client, bucket) + + const ( + includePattern = "*.py" + fileContent = "content" + ) + + files := [...]string{ + "file1.py", + "file2.py", + "file.txt", + "data.txt", + "src/app.py", + } + filesKept := [...]string{ + "file.txt", + "data.txt", + } + + for _, filename := range files { + putFile(t, s3client, bucket, filename, fileContent) + } + + srcpath := fmt.Sprintf("s3://%s", bucket) + + cmd := s5cmd("rm", "--include", includePattern, srcpath+"/*") + result := icmd.RunCmd(cmd) + + result.Assert(t, icmd.Success) + + fmt.Println(result.Stdout()) + + assertLines(t, result.Stdout(), map[int]compareFunc{ + 0: equals("rm %v/%s", srcpath, files[0]), + 1: equals("rm %v/%s", srcpath, files[1]), + 2: equals("rm %v/%s", srcpath, files[4]), + }, sortInput(true)) + + // assert s3 + for _, f := range filesKept { + assert.Assert(t, ensureS3Object(s3client, bucket, f, fileContent)) + } +} + +// rm --include "file*" --exclude "*.py" s3://bucket/ +func TestRemoveS3ObjectsWithIncludeExcludeFilter(t *testing.T) { + t.Parallel() + + s3client, s5cmd := setup(t) + + bucket := s3BucketFromTestName(t) + createBucket(t, s3client, bucket) + + const ( + includePattern = "*.md" + excludePattern = "*.py" + fileContent = "content" + ) + + files := [...]string{ + "file1.py", + "file2.py", + "test.py", + "app.py", + "docs/file.md", + } + filesKept := [...]string{ + "file1.py", + "file2.py", + "test.py", + "app.py", + } + + for _, filename := range files { + putFile(t, s3client, bucket, filename, fileContent) + } + + srcpath := fmt.Sprintf("s3://%s", bucket) + + cmd := s5cmd("rm", "--include", includePattern, "--exclude", excludePattern, srcpath+"/*") + result := icmd.RunCmd(cmd) + + result.Assert(t, icmd.Success) + + assertLines(t, result.Stdout(), map[int]compareFunc{ + 0: equals("rm %v/%s", srcpath, files[4]), + }, sortInput(true)) + + // assert s3 + for _, f := range filesKept { + assert.Assert(t, ensureS3Object(s3client, bucket, f, fileContent)) + } +} + +// rm --exclude "docs*" --include "*.md" --include "*.py" s3://bucket/ +func TestRemoveS3ObjectsWithIncludeExcludeFilter2(t *testing.T) { + t.Parallel() + + s3client, s5cmd := setup(t) + + bucket := s3BucketFromTestName(t) + createBucket(t, s3client, bucket) + + const ( + includePattern = "*.md" + includePattern2 = "*.py" + excludePattern = "docs*" + fileContent = "content" + ) + + files := [...]string{ + "file1.py", + "file2.py", + "test.py", + "app.py", + "docs/readme.md", + } + filesKept := [...]string{ + "docs/readme.md", + } + + for _, filename := range files { + putFile(t, s3client, bucket, filename, fileContent) + } + + srcpath := fmt.Sprintf("s3://%s", bucket) + + cmd := s5cmd("rm", "--exclude", excludePattern, "--include", includePattern, "--include", includePattern2, srcpath+"/*") + result := icmd.RunCmd(cmd) + + result.Assert(t, icmd.Success) + + assertLines(t, result.Stdout(), map[int]compareFunc{ + 0: equals("rm %v/%s", srcpath, files[3]), + 1: equals("rm %v/%s", srcpath, files[0]), + 2: equals("rm %v/%s", srcpath, files[1]), + 3: equals("rm %v/%s", srcpath, files[2]), + }, sortInput(true)) + + // assert s3 + for _, f := range filesKept { + assert.Assert(t, ensureS3Object(s3client, bucket, f, fileContent)) + } +} diff --git a/e2e/sync_test.go b/e2e/sync_test.go index 3cc2894af..077034006 100644 --- a/e2e/sync_test.go +++ b/e2e/sync_test.go @@ -2,6 +2,8 @@ package e2e import ( "fmt" + "net" + "os" "path/filepath" "runtime" "testing" @@ -1793,3 +1795,256 @@ func TestIssue435(t *testing.T) { assertError(t, err, errS3NoSuchKey) } } + +// sync s3://bucket/* s3://bucket/ (dest bucket is empty) +func TestSyncS3BucketToEmptyS3BucketWithExitOnErrorFlag(t *testing.T) { + t.Parallel() + s3client, s5cmd := setup(t) + + bucket := s3BucketFromTestName(t) + dstbucket := s3BucketFromTestNameWithPrefix(t, "dst") + + const ( + prefix = "prefix" + ) + createBucket(t, s3client, bucket) + createBucket(t, s3client, dstbucket) + + S3Content := map[string]string{ + "testfile.txt": "S: this is a test file", + "readme.md": "S: this is a readme file", + "a/another_test_file.txt": "S: yet another txt file", + "abc/def/test.py": "S: file in nested folders", + } + + for filename, content := range S3Content { + putFile(t, s3client, bucket, filename, content) + } + + bucketPath := fmt.Sprintf("s3://%v", bucket) + src := fmt.Sprintf("%v/*", bucketPath) + dst := fmt.Sprintf("s3://%v/%v/", dstbucket, prefix) + + cmd := s5cmd("sync", "--exit-on-error", src, dst) + result := icmd.RunCmd(cmd) + + result.Assert(t, icmd.Success) + + assertLines(t, result.Stdout(), map[int]compareFunc{ + 0: equals(`cp %v/a/another_test_file.txt %va/another_test_file.txt`, bucketPath, dst), + 1: equals(`cp %v/abc/def/test.py %vabc/def/test.py`, bucketPath, dst), + 2: equals(`cp %v/readme.md %vreadme.md`, bucketPath, dst), + 3: equals(`cp %v/testfile.txt %vtestfile.txt`, bucketPath, dst), + }, sortInput(true)) + + // assert s3 objects in source bucket. + for key, content := range S3Content { + assert.Assert(t, ensureS3Object(s3client, bucket, key, content)) + } + + // assert s3 objects in dest bucket + for key, content := range S3Content { + key = fmt.Sprintf("%s/%s", prefix, key) // add the prefix + assert.Assert(t, ensureS3Object(s3client, dstbucket, key, content)) + } +} + +// sync --exit-on-error s3://bucket/* s3://NotExistingBucket/ (dest bucket doesn't exist) +func TestSyncExitOnErrorS3BucketToS3BucketThatDoesNotExist(t *testing.T) { + t.Parallel() + + now := time.Now() + timeSource := newFixedTimeSource(now) + s3client, s5cmd := setup(t, withTimeSource(timeSource)) + + bucket := s3BucketFromTestName(t) + destbucket := "NotExistingBucket" + + createBucket(t, s3client, bucket) + + S3Content := map[string]string{ + "testfile.txt": "S: this is a test file", + "readme.md": "S: this is a readme file", + "a/another_test_file.txt": "S: yet another txt file", + "abc/def/test.py": "S: file in nested folders", + } + + for filename, content := range S3Content { + putFile(t, s3client, bucket, filename, content) + } + + src := fmt.Sprintf("s3://%v/*", bucket) + dst := fmt.Sprintf("s3://%v/", destbucket) + + cmd := s5cmd("sync", "--exit-on-error", src, dst) + result := icmd.RunCmd(cmd) + + result.Assert(t, icmd.Expected{ExitCode: 1}) + + assertLines(t, result.Stderr(), map[int]compareFunc{ + 0: contains(`status code: 404`), + }) +} + +// sync s3://bucket/* s3://NotExistingBucket/ (dest bucket doesn't exist) +func TestSyncS3BucketToS3BucketThatDoesNotExist(t *testing.T) { + t.Parallel() + + now := time.Now() + timeSource := newFixedTimeSource(now) + s3client, s5cmd := setup(t, withTimeSource(timeSource)) + + bucket := s3BucketFromTestName(t) + destbucket := "NotExistingBucket" + + createBucket(t, s3client, bucket) + + S3Content := map[string]string{ + "testfile.txt": "S: this is a test file", + "readme.md": "S: this is a readme file", + "a/another_test_file.txt": "S: yet another txt file", + "abc/def/test.py": "S: file in nested folders", + } + + for filename, content := range S3Content { + putFile(t, s3client, bucket, filename, content) + } + + src := fmt.Sprintf("s3://%v/*", bucket) + dst := fmt.Sprintf("s3://%v/", destbucket) + + cmd := s5cmd("sync", src, dst) + result := icmd.RunCmd(cmd) + + result.Assert(t, icmd.Expected{ExitCode: 1}) + + assertLines(t, result.Stderr(), map[int]compareFunc{ + 0: contains(`status code: 404`), + }) +} + +// If source path contains a special file it should not be synced +func TestSyncSocketDestinationEmpty(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip() + } + + t.Parallel() + + s3client, s5cmd := setup(t) + bucket := s3BucketFromTestName(t) + createBucket(t, s3client, bucket) + + workdir := fs.NewDir(t, t.Name()) + defer workdir.Remove() + + sockaddr := workdir.Join("/s5cmd.sock") + ln, err := net.Listen("unix", sockaddr) + if err != nil { + t.Fatal(err) + } + + t.Cleanup(func() { + ln.Close() + os.Remove(sockaddr) + }) + + cmd := s5cmd("sync", ".", "s3://"+bucket+"/") + result := icmd.RunCmd(cmd, withWorkingDir(workdir)) + + // assert error message + assertLines(t, result.Stderr(), map[int]compareFunc{ + 0: contains(`is not a regular file`), + }) + + // assert logs are empty (no sync) + assertLines(t, result.Stdout(), nil) + + // assert exit code + result.Assert(t, icmd.Expected{ExitCode: 1}) +} + +// sync --include pattern s3://bucket/* s3://anotherbucket/prefix/ +func TestSyncS3ObjectsIntoAnotherBucketWithIncludeFilters(t *testing.T) { + t.Parallel() + + srcbucket := s3BucketFromTestNameWithPrefix(t, "src") + dstbucket := s3BucketFromTestNameWithPrefix(t, "dst") + + s3client, s5cmd := setup(t) + + createBucket(t, s3client, srcbucket) + createBucket(t, s3client, dstbucket) + + srcFiles := []string{ + "file_already_exists_in_destination.txt", + "file_not_exists_in_destination.txt", + "main.py", + "main.js", + "readme.md", + "main.pdf", + "main/file.txt", + } + + dstFiles := []string{ + "prefix/file_already_exists_in_destination.txt", + } + + excludedFiles := []string{ + "prefix/file_not_exists_in_destination.txt", + } + + includedFiles := []string{ + "main.js", + "main.pdf", + "main.py", + "main/file.txt", + "readme.md", + } + + const ( + content = "this is a file content" + includePattern1 = "main*" + includePattern2 = "*.md" + ) + + for _, filename := range srcFiles { + putFile(t, s3client, srcbucket, filename, content) + } + + for _, filename := range dstFiles { + putFile(t, s3client, dstbucket, filename, content) + } + + src := fmt.Sprintf("s3://%v/*", srcbucket) + dst := fmt.Sprintf("s3://%v/prefix/", dstbucket) + + cmd := s5cmd("sync", "--include", includePattern1, "--include", includePattern2, src, dst) + result := icmd.RunCmd(cmd) + + result.Assert(t, icmd.Success) + + assertLines(t, result.Stdout(), map[int]compareFunc{ + 0: equals(`cp s3://%s/%s s3://%s/prefix/%s`, srcbucket, includedFiles[0], dstbucket, includedFiles[0]), + 1: equals(`cp s3://%s/%s s3://%s/prefix/%s`, srcbucket, includedFiles[1], dstbucket, includedFiles[1]), + 2: equals(`cp s3://%s/%s s3://%s/prefix/%s`, srcbucket, includedFiles[2], dstbucket, includedFiles[2]), + 3: equals(`cp s3://%s/%s s3://%s/prefix/%s`, srcbucket, includedFiles[3], dstbucket, includedFiles[3]), + 4: equals(`cp s3://%s/%s s3://%s/prefix/%s`, srcbucket, includedFiles[4], dstbucket, includedFiles[4]), + }, sortInput(true)) + + // assert s3 source objects + for _, filename := range srcFiles { + assert.Assert(t, ensureS3Object(s3client, srcbucket, filename, content)) + } + + // assert s3 destination objects + for _, filename := range includedFiles { + assert.Assert(t, ensureS3Object(s3client, dstbucket, "prefix/"+filename, content)) + } + + // assert s3 destination objects which should not be in bucket. + for _, filename := range excludedFiles { + err := ensureS3Object(s3client, dstbucket, filename, content) + assertError(t, err, errS3NoSuchKey) + } +} diff --git a/progressbar/progressbar_test.go b/progressbar/progressbar_test.go index f9cf8cff8..8c8a12f9f 100644 --- a/progressbar/progressbar_test.go +++ b/progressbar/progressbar_test.go @@ -38,8 +38,7 @@ func TestCommandProgress_AddCompletedBytes(t *testing.T) { cp.Start() bytes := int64(101) cp.AddCompletedBytes(bytes) - assert.Equal(t, int64(101), cp.progressbar.Current()) - assert.Equal(t, int64(bytes), cp.progressbar.Current()) + assert.Equal(t, bytes, cp.progressbar.Current()) assert.Equal(t, true, strings.Contains(cp.progressbar.String(), "101 B")) } @@ -49,7 +48,6 @@ func TestCommandProgress_AddTotalBytes(t *testing.T) { cp.Start() bytes := int64(102) cp.AddTotalBytes(bytes) - assert.Equal(t, int64(102), cp.progressbar.Total()) assert.Equal(t, bytes, cp.progressbar.Total()) assert.Equal(t, true, strings.Contains(cp.progressbar.String(), "102 B")) } diff --git a/storage/storage.go b/storage/storage.go index ea340f649..e20332636 100644 --- a/storage/storage.go +++ b/storage/storage.go @@ -168,6 +168,11 @@ func (o ObjectType) IsSymlink() bool { return o.mode&os.ModeSymlink != 0 } +// IsRegular checks if the object is a regular file. +func (o ObjectType) IsRegular() bool { + return o.mode.IsRegular() +} + // ShouldProcessURL returns true if follow symlinks is enabled. // If follow symlinks is disabled we should not process the url. // (this check is needed only for local files)