From 33fac8b7275b89e2f0dfcccab706e0ad7dd45fc8 Mon Sep 17 00:00:00 2001 From: Alexandr Bakurin Date: Mon, 29 May 2023 22:21:30 +0200 Subject: [PATCH] add Exclude support for Delete and Copy/Mcopy commands --- README.md | 9 +- pkg/config/command.go | 14 +- pkg/executor/dry.go | 32 +++- pkg/executor/executor.go | 24 ++- pkg/executor/local.go | 90 ++++++++- pkg/executor/local_test.go | 209 +++++++++++++++++++++ pkg/executor/remote.go | 122 +++++++++++- pkg/executor/remote_test.go | 59 +++++- pkg/executor/testdata/data3.txt | 1 + pkg/executor/testdata/delete/d1/file11.txt | 1 + pkg/executor/testdata/delete/d1/file12.txt | 1 + pkg/executor/testdata/delete/d2/file21.txt | 1 + pkg/executor/testdata/delete/d2/file22.txt | 1 + pkg/executor/testdata/delete/file1.txt | 6 + pkg/executor/testdata/delete/file2.txt | 3 + pkg/executor/testdata/delete/file3.txt | 2 + pkg/runner/commands.go | 4 +- 17 files changed, 546 insertions(+), 33 deletions(-) create mode 100644 pkg/executor/testdata/data3.txt create mode 100644 pkg/executor/testdata/delete/d1/file11.txt create mode 100644 pkg/executor/testdata/delete/d1/file12.txt create mode 100644 pkg/executor/testdata/delete/d2/file21.txt create mode 100644 pkg/executor/testdata/delete/d2/file22.txt create mode 100644 pkg/executor/testdata/delete/file1.txt create mode 100644 pkg/executor/testdata/delete/file2.txt create mode 100644 pkg/executor/testdata/delete/file3.txt diff --git a/README.md b/README.md index 7e74034e..ba72122d 100644 --- a/README.md +++ b/README.md @@ -333,7 +333,7 @@ script: | Copies a file from the local machine to the remote host(s). If `mkdir` is set to `true` the command will create the destination directory if it doesn't exist, same as `mkdir -p` in bash. The command also supports glob patterns in `src` field. Copy command performs a quick check to see if the file already exists on the remote host(s) with the same size and modification time, -and skips the copy if it does. This option can be disabled by setting `force: true` flag. +and skips the copy if it does. This option can be disabled by setting `force: true` flag. Another option is `exclude` which allows to specify a list of files to exclude to be copied. ```yaml - name: copy file with mkdir @@ -342,6 +342,8 @@ and skips the copy if it does. This option can be disabled by setting `force: tr - name: copy files with glob copy: {"src": "testdata/*.csv", "dst": "/tmp/things"} +- name: copy files with glob and exclude + copy: {"src": "testdata/*.yml", "dst": "/tmp/things", "exclude": ["conf.dist.yml"]} - name: copy files with force flag copy: {"src": "testdata/*.csv", "dst": "/tmp/things", "force": true} @@ -360,7 +362,6 @@ Copy also supports list format to copy multiple files at once: Synchronises directory from the local machine to the remote host(s). Optionally supports deleting files on the remote host(s) that don't exist locally with `"delete": true` flag. Another option is `exclude` which allows to specify a list of files to exclude from the sync. - ```yaml - name: sync directory sync: {"src": "testdata", "dst": "/tmp/things"} @@ -382,8 +383,12 @@ Deletes a file or directory on the remote host(s), optionally can remove recursi ```yaml - name: delete file delete: {"path": "/tmp/things.csv"} + - name: delete directory recursively delete: {"path": "/tmp/things", "recur": true} + +- name: delete directory recursively with exclude + delete: {"path": "/tmp/things", "recur": true, "exclude": ["*.txt", "*.yml"]} ``` Delete also supports list format to remove multiple paths at once. diff --git a/pkg/config/command.go b/pkg/config/command.go index 31d27ef5..ef26e628 100644 --- a/pkg/config/command.go +++ b/pkg/config/command.go @@ -44,10 +44,11 @@ type CmdOptions struct { // CopyInternal defines copy command, implemented internally type CopyInternal struct { - Source string `yaml:"src" toml:"src"` // source must be a file or a glob pattern - Dest string `yaml:"dst" toml:"dst"` // destination must be a file or a directory - Mkdir bool `yaml:"mkdir" toml:"mkdir"` // create destination directory if it does not exist - Force bool `yaml:"force" toml:"force"` // force copy even if source and destination are the same + Source string `yaml:"src" toml:"src"` // source must be a file or a glob pattern + Dest string `yaml:"dst" toml:"dst"` // destination must be a file or a directory + Mkdir bool `yaml:"mkdir" toml:"mkdir"` // create destination directory if it does not exist + Force bool `yaml:"force" toml:"force"` // force copy even if source and destination are the same + Exclude []string `yaml:"exclude" toml:"exclude"` // exclude files matching these patterns } // SyncInternal defines sync command (recursive copy), implemented internally @@ -61,8 +62,9 @@ type SyncInternal struct { // DeleteInternal defines delete command, implemented internally type DeleteInternal struct { - Location string `yaml:"path" toml:"path"` - Recursive bool `yaml:"recur" toml:"recur"` + Location string `yaml:"path" toml:"path"` + Recursive bool `yaml:"recur" toml:"recur"` + Exclude []string `yaml:"exclude" toml:"exclude"` } // WaitInternal defines wait command, implemented internally diff --git a/pkg/executor/dry.go b/pkg/executor/dry.go index ee912d98..f8a0b0d6 100644 --- a/pkg/executor/dry.go +++ b/pkg/executor/dry.go @@ -45,8 +45,15 @@ func (ex *Dry) Run(_ context.Context, cmd string, opts *RunOpts) (out []string, // Upload doesn't actually upload, just prints the command func (ex *Dry) Upload(_ context.Context, local, remote string, opts *UpDownOpts) (err error) { - mkdir := opts != nil && opts.Mkdir - log.Printf("[DEBUG] upload %s to %s, mkdir: %v", local, remote, mkdir) + var mkdir bool + var exclude []string + + if opts != nil { + mkdir = opts.Mkdir + exclude = opts.Exclude + } + + log.Printf("[DEBUG] upload %s to %s, mkdir: %v, exclude: %v", local, remote, mkdir, exclude) if strings.Contains(remote, "spot-script") { // this is a temp script created by spot to perform script execution on remote host outLog, outErr := MakeOutAndErrWriters(ex.hostAddr, ex.hostName, true, ex.secrets) @@ -71,8 +78,15 @@ func (ex *Dry) Upload(_ context.Context, local, remote string, opts *UpDownOpts) // Download file from remote server with scp func (ex *Dry) Download(_ context.Context, remote, local string, opts *UpDownOpts) (err error) { - mkdir := opts != nil && opts.Mkdir - log.Printf("[DEBUG] download %s to %s, mkdir: %v", local, remote, mkdir) + var mkdir bool + var exclude []string + + if opts != nil { + mkdir = opts.Mkdir + exclude = opts.Exclude + } + + log.Printf("[DEBUG] download %s to %s, mkdir: %v, exclude: %v", local, remote, mkdir, exclude) return nil } @@ -89,8 +103,14 @@ func (ex *Dry) Sync(_ context.Context, localDir, remoteDir string, opts *SyncOpt // Delete doesn't delete anything, just prints the command func (ex *Dry) Delete(_ context.Context, remoteFile string, opts *DeleteOpts) (err error) { - recursive := opts != nil && opts.Recursive - log.Printf("[DEBUG] delete %s, recursive: %v", remoteFile, recursive) + var recursive bool + var exclude []string + + if opts != nil { + recursive = opts.Recursive + exclude = opts.Exclude + } + log.Printf("[DEBUG] delete %s, recursive: %v, exclude: %v", remoteFile, recursive, exclude) return nil } diff --git a/pkg/executor/executor.go b/pkg/executor/executor.go index 809c04e2..9709a2ce 100644 --- a/pkg/executor/executor.go +++ b/pkg/executor/executor.go @@ -37,9 +37,10 @@ type RunOpts struct { // UpDownOpts is a struct for upload and download options. type UpDownOpts struct { - Mkdir bool // create remote directory if it does not exist - Checksum bool // compare checksums of local and remote files, default is size and modtime - Force bool // overwrite existing files on remote + Mkdir bool // create remote directory if it does not exist + Checksum bool // compare checksums of local and remote files, default is size and modtime + Force bool // overwrite existing files on remote + Exclude []string // exclude files matching the given patterns } // SyncOpts is a struct for sync options. @@ -52,7 +53,8 @@ type SyncOpts struct { // DeleteOpts is a struct for delete options. type DeleteOpts struct { - Recursive bool // delete directories recursively + Recursive bool // delete directories recursively + Exclude []string // exclude files matching the given patterns } // StdOutLogWriter is a writer that writes to log with a prefix and a log level. @@ -184,6 +186,20 @@ func isExcluded(path string, excl []string) bool { return false } +func isExcludedSubPath(path string, excl []string) bool { + subpath := filepath.Join(path, "*") + for _, ex := range excl { + match, err := filepath.Match(subpath, strings.TrimSuffix(ex, "/*")) + if err != nil { + continue + } + if match { + return true + } + } + return false +} + func isWithinOneSecond(t1, t2 time.Time) bool { diff := t1.Sub(t2) if diff < 0 { diff --git a/pkg/executor/local.go b/pkg/executor/local.go index 1f27eb96..d1518601 100644 --- a/pkg/executor/local.go +++ b/pkg/executor/local.go @@ -60,13 +60,29 @@ func (l *Local) Upload(_ context.Context, src, dst string, opts *UpDownOpts) (er return fmt.Errorf("source file %q not found", src) } - if opts != nil && opts.Mkdir { + var mkdir bool + var exclude []string + + if opts != nil { + mkdir = opts.Mkdir + exclude = opts.Exclude + } + + if mkdir { if err = os.MkdirAll(filepath.Dir(dst), 0o750); err != nil { return fmt.Errorf("can't create local dir %s: %w", filepath.Dir(dst), err) } } for _, match := range matches { + relPath, e := filepath.Rel(filepath.Dir(src), match) + if e != nil { + return fmt.Errorf("failed to build relative path for %s: %w", match, err) + } + if isExcluded(relPath, exclude) { + continue + } + destination := dst if len(matches) > 1 { destination = filepath.Join(dst, filepath.Base(match)) @@ -128,12 +144,18 @@ func (l *Local) Sync(ctx context.Context, src, dst string, opts *SyncOpts) ([]st } // Delete file or directory -func (l *Local) Delete(_ context.Context, remoteFile string, opts *DeleteOpts) (err error) { +func (l *Local) Delete(ctx context.Context, remoteFile string, opts *DeleteOpts) (err error) { recursive := opts != nil && opts.Recursive if !recursive { return os.Remove(remoteFile) } - return os.RemoveAll(remoteFile) + + var exclude []string + if opts != nil { + exclude = opts.Exclude + } + + return l.deletePath(ctx, remoteFile, exclude) } // Close does nothing for local @@ -259,3 +281,65 @@ func (l *Local) copyFile(src, dst string) error { return nil } + +func (l *Local) deletePath(ctx context.Context, src string, excl []string) error { + info, err := os.Stat(src) + if err != nil { + return err + } + + if !info.IsDir() || len(excl) == 0 { + return os.RemoveAll(src) + } + + hasExclusion := false + err = filepath.Walk(src, func(filePath string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + if ctx.Err() != nil { + return ctx.Err() + } + + relPath, err := filepath.Rel(src, filePath) + if err != nil { + return err + } + + if info.IsDir() && isExcludedSubPath(relPath, excl) { + return nil + } + + if isExcluded(relPath, excl) { + hasExclusion = true + if info.IsDir() { + return filepath.SkipDir + } + + return nil + } + + if !info.IsDir() { + return os.Remove(filePath) + } + + err = os.RemoveAll(filePath) + if err != nil { + return err + } + + return filepath.SkipDir + }) + + if err != nil { + return err + } + + // remove the whole directory if there are no actual exclusions + if !hasExclusion { + return os.RemoveAll(src) + } + + return nil +} diff --git a/pkg/executor/local_test.go b/pkg/executor/local_test.go index 459f7eac..3d4dc0a7 100644 --- a/pkg/executor/local_test.go +++ b/pkg/executor/local_test.go @@ -319,6 +319,86 @@ func TestUploadDownloadWithGlob(t *testing.T) { } } +func TestUploadDownloadWithExclude(t *testing.T) { + l := &Local{} + + for _, tc := range []struct { + name string + src string + dst string + mkdir bool + excl []string + dstStructure map[string]string + expectError bool + }{ + { + name: "successful upload with mkdir=true and excluded file", + src: "*.txt", + dstStructure: map[string]string{ + "data1.txt": "data1 content", + }, + mkdir: true, + excl: []string{"data2.txt"}, + }, + { + name: "successful upload with mkdir=true and excluded glob", + src: "*.txt", + dstStructure: map[string]string{ + "data2.txt": "data2 content", + }, + mkdir: true, + excl: []string{"data1.*"}, + }, + } { + + t.Run(fmt.Sprintf("%s#%s", tc.name, tc.name), func(t *testing.T) { + // create some temporary test files with content + tmpDir, err := os.MkdirTemp("", "test") + require.NoError(t, err) + defer os.RemoveAll(tmpDir) + + data1File := filepath.Join(tmpDir, "data1.txt") + err = os.WriteFile(data1File, []byte("data1 content"), 0o644) + require.NoError(t, err) + + data2File := filepath.Join(tmpDir, "data2.txt") + err = os.WriteFile(data2File, []byte("data2 content"), 0o644) + require.NoError(t, err) + + // create a temporary destination directory + dstDir, err := os.MkdirTemp("", "dst") + require.NoError(t, err) + defer os.RemoveAll(dstDir) + + if tc.src != "" { + dstDir = filepath.Join(dstDir, tc.dst) + } + + err = l.Upload(context.Background(), filepath.Join(tmpDir, tc.src), dstDir, &UpDownOpts{Mkdir: tc.mkdir, Exclude: tc.excl}) + + if tc.expectError { + assert.Error(t, err, "expected an error") + return + } + + assert.NoError(t, err, "unexpected error") + + // assert that all files were uploaded + files, err := os.ReadDir(dstDir) + require.NoError(t, err) + assert.Len(t, files, len(tc.dstStructure), "unexpected number of uploaded files") + + // assert that the contents of the uploaded files match the contents of the source files + for name, content := range tc.dstStructure { + dstContent, err := os.ReadFile(filepath.Join(dstDir, name)) + require.NoError(t, err) + assert.Equal(t, content, string(dstContent), "uploaded content should match source content") + } + }) + + } +} + func TestLocal_Sync(t *testing.T) { testCases := []struct { @@ -622,6 +702,135 @@ func TestDelete(t *testing.T) { } } +func TestDeleteWithExclude(t *testing.T) { + type testCase struct { + name string + recursive bool + isDir bool + srcStructure map[string]bool + dstStructure map[string]bool + expectError bool + exclude []string + } + + testCases := []testCase{ + { + name: "successful delete directory with recursive=true and excluded files", + isDir: true, + srcStructure: map[string]bool{ + "file1.txt": false, + "file2.yaml": false, + "file2.toml": false, + "dir1/file3.txt": false, + "dir1/file4.txt": false, + "dir2/dir3/": true, + "dir4/dir5/": true, + }, + dstStructure: map[string]bool{ + "file1.txt": false, + "file2.yaml": false, + "file2.toml": false, + "dir1/file3.txt": false, + "dir2/dir3/": true, + }, + recursive: true, + exclude: []string{ + "file1.txt", + "file2.*", + "dir1/file3.txt", + "dir2/*", + }, + }, + { + name: "successful delete directory with recursive=true and non-existing excluded file", + isDir: true, + srcStructure: map[string]bool{ + "file1.txt": false, + "dir1/file2.txt": false, + "dir1/file3.txt": false, + "dir2/dir3/*": true, + }, + dstStructure: map[string]bool{}, + recursive: true, + exclude: []string{ + "dir5/file2.txt", + }, + }, + { + name: "successfully delete file with recursive=true and defined exclusion list", + isDir: false, + srcStructure: map[string]bool{}, + dstStructure: map[string]bool{}, + recursive: true, + expectError: false, + exclude: []string{ + "file1.txt", + }, + }, + } + + initTestCase := func(tc testCase) (string, error) { + if tc.isDir { + remoteFile, err := os.MkdirTemp("", "test") + require.NoError(t, err) + + for subPath, isDir := range tc.srcStructure { + err = os.MkdirAll(filepath.Join(remoteFile, filepath.Dir(subPath)), 0o700) + if err != nil { + return "", err + } + + if !isDir { + err = os.WriteFile(filepath.Join(remoteFile, subPath), []byte(""), 0o644) + if err != nil { + return "", err + } + } + } + + return remoteFile, nil + } + + require.Empty(t, tc.srcStructure, "structure can be defined for directory only") + tempFile, e := os.CreateTemp("", "test") + require.NoError(t, e) + err := tempFile.Close() + require.NoError(t, err) + + return tempFile.Name(), nil + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + remoteFile, err := initTestCase(tc) + assert.NoError(t, err, "unable to initialize test case") + + l := &Local{} + err = l.Delete(context.Background(), remoteFile, &DeleteOpts{Recursive: tc.recursive, Exclude: tc.exclude}) + if tc.expectError { + assert.Error(t, err, "expected an error") + } else { + assert.NoError(t, err, "unexpected error") + + if len(tc.dstStructure) == 0 { + _, err := os.Stat(remoteFile) + assert.True(t, os.IsNotExist(err), "remote file should be deleted") + } else { + for src := range tc.srcStructure { + _, shouldExist := tc.dstStructure[src] + _, err := os.Stat(filepath.Join(remoteFile, src)) + if shouldExist { + assert.NoError(t, err, "remote file should not be deleted") + } else { + assert.True(t, os.IsNotExist(err), "remote file should be deleted") + } + } + } + } + }) + } +} + func TestClose(t *testing.T) { l := &Local{} err := l.Close() diff --git a/pkg/executor/remote.go b/pkg/executor/remote.go index 5e8b282b..bfbd8755 100644 --- a/pkg/executor/remote.go +++ b/pkg/executor/remote.go @@ -71,8 +71,21 @@ func (ex *Remote) Upload(ctx context.Context, local, remote string, opts *UpDown return fmt.Errorf("source file %q not found", local) } + var exclude []string + if opts != nil { + exclude = opts.Exclude + } + // upload each file matching the glob pattern. If no glob pattern is found, the file is matched as is for _, match := range matches { + relPath, e := filepath.Rel(filepath.Dir(local), match) + if e != nil { + return fmt.Errorf("failed to build relative path for %s: %w", match, err) + } + if isExcluded(relPath, exclude) { + continue + } + remoteFile := remote if len(matches) > 1 { // if there are multiple files, treat remote as a directory remoteFile = filepath.Join(remote, filepath.Base(match)) @@ -105,15 +118,43 @@ func (ex *Remote) Download(ctx context.Context, remote, local string, opts *UpDo return fmt.Errorf("failed to split hostAddr and port: %w", err) } - req := sftpReq{ - client: ex.client, - localFile: local, - remoteFile: remote, - mkdir: opts != nil && opts.Mkdir, - remoteHost: host, - remotePort: port, + var mkdir bool + var exclude []string + + if opts != nil { + mkdir = opts.Mkdir + exclude = opts.Exclude + } + + remoteFiles, err := ex.findMatchedFiles(remote, exclude) + if err != nil { + return fmt.Errorf("failed to list remote files by glob for %s: %w", remote, err) + } + + for _, remoteFile := range remoteFiles { + localFile := local + + // if the remote basename does not equal the remoteFile basename, + // treat remote as a glob pattern and local as a directory + if filepath.Base(remote) != filepath.Base(remoteFile) { + localFile = filepath.Join(local, filepath.Base(remoteFile)) + } + + req := sftpReq{ + client: ex.client, + localFile: localFile, + remoteFile: remoteFile, + mkdir: mkdir, + remoteHost: host, + remotePort: port, + } + err = ex.sftpDownload(ctx, req) + if err != nil { + return fmt.Errorf("failed to download remote file %s: %w", remoteFile, err) + } } - return ex.sftpDownload(ctx, req) + + return nil } // Sync compares local and remote files and uploads unmatched files, recursively. @@ -178,8 +219,16 @@ func (ex *Remote) Delete(ctx context.Context, remoteFile string, opts *DeleteOpt return fmt.Errorf("failed to stat %s: %w", remoteFile, err) } - recursive := opts != nil && opts.Recursive + var recursive bool + var exclude []string + + if opts != nil { + recursive = opts.Recursive + exclude = opts.Exclude + } + if fileInfo.IsDir() && recursive { + hasExclusion := false walker := sftpClient.Walk(remoteFile) var pathsToDelete []string @@ -187,9 +236,35 @@ func (ex *Remote) Delete(ctx context.Context, remoteFile string, opts *DeleteOpt if walker.Err() != nil { continue } + + path := walker.Path() + relPath, e := filepath.Rel(remoteFile, path) + if e != nil { + return e + } + + // skip parent directories of the excluded files + if walker.Stat().IsDir() && (relPath == "." || isExcludedSubPath(relPath, exclude)) { + continue + } + + if isExcluded(relPath, exclude) { + hasExclusion = true + if walker.Stat().IsDir() { + walker.SkipDir() + } + + continue + } + pathsToDelete = append(pathsToDelete, walker.Path()) } + // there is no actual exclusions + if !hasExclusion { + pathsToDelete = append([]string{remoteFile}, pathsToDelete...) + } + // Delete files and directories in reverse order for i := len(pathsToDelete) - 1; i >= 0; i-- { select { @@ -358,6 +433,7 @@ func (ex *Remote) sftpUpload(ctx context.Context, req sftpReq) error { return nil } + func (ex *Remote) sftpDownload(ctx context.Context, req sftpReq) error { log.Printf("[INFO] download %s from %s:%s", req.localFile, req.remoteHost, req.remoteFile) defer func(st time.Time) { log.Printf("[DEBUG] download done for %q in %s", req.localFile, time.Since(st)) }(time.Now()) @@ -553,3 +629,31 @@ func (ex *Remote) findUnmatchedFiles(local, remote map[string]fileProperties, ex return updatedFiles, deletedFiles } + +func (ex *Remote) findMatchedFiles(remote string, excl []string) ([]string, error) { + sftpClient, err := sftp.NewClient(ex.client) + if err != nil { + return nil, fmt.Errorf("failed to create sftp client: %v", err) + } + defer sftpClient.Close() + + matches, err := sftpClient.Glob(remote) + if err != nil { + return nil, fmt.Errorf("failed to list remote files: %v", err) + } + + files := make([]string, 0, len(matches)) + for _, match := range matches { + relPath, e := filepath.Rel(filepath.Dir(remote), match) + if e != nil { + return nil, fmt.Errorf("failed to build relative path for %s: %w", match, err) + } + if isExcluded(relPath, excl) { + continue + } + + files = append(files, match) + } + + return files, nil +} diff --git a/pkg/executor/remote_test.go b/pkg/executor/remote_test.go index 89f1f09d..2763b44a 100644 --- a/pkg/executor/remote_test.go +++ b/pkg/executor/remote_test.go @@ -7,6 +7,7 @@ import ( "io" "log" "os" + "path/filepath" "sort" "testing" "time" @@ -58,7 +59,7 @@ func TestExecuter_UploadGlobAndDownload(t *testing.T) { require.NoError(t, err) defer sess.Close() - err = sess.Upload(ctx, "testdata/data*.txt", "/tmp/blah", &UpDownOpts{Mkdir: true}) + err = sess.Upload(ctx, "testdata/data*.txt", "/tmp/blah", &UpDownOpts{Mkdir: true, Exclude: []string{"data3.txt"}}) require.NoError(t, err) { @@ -87,6 +88,21 @@ func TestExecuter_UploadGlobAndDownload(t *testing.T) { require.NoError(t, err) assert.Equal(t, string(exp), string(act)) } + { + tmpDir, err := os.MkdirTemp("", "test") + require.NoError(t, err) + defer os.RemoveAll(tmpDir) + err = sess.Download(ctx, "/tmp/blah/data*.txt", tmpDir, &UpDownOpts{Mkdir: true, Exclude: []string{"data2.txt"}}) + require.NoError(t, err) + assert.FileExists(t, filepath.Join(tmpDir, "data1.txt")) + exp, err := os.ReadFile("testdata/data1.txt") + require.NoError(t, err) + act, err := os.ReadFile(filepath.Join(tmpDir, "data1.txt")) + require.NoError(t, err) + assert.Equal(t, string(exp), string(act)) + assert.NoFileExists(t, filepath.Join(tmpDir, "data2.txt"), "remote file should be downloaded") + assert.NoFileExists(t, filepath.Join(tmpDir, "data3.txt"), "remote file should be downloaded") + } } func TestExecuter_Upload_FailedSourceNotFound(t *testing.T) { @@ -432,6 +448,47 @@ func TestExecuter_Delete(t *testing.T) { }) } +func TestExecuter_DeleteWithExclude(t *testing.T) { + ctx := context.Background() + hostAndPort, teardown := startTestContainer(t) + defer teardown() + + c, err := NewConnector("testdata/test_ssh_key", time.Second*10) + require.NoError(t, err) + sess, err := c.Connect(ctx, hostAndPort, "h1", "test") + require.NoError(t, err) + defer sess.Close() + + res, err := sess.Sync(ctx, "testdata/delete", "/tmp/delete.dest", &SyncOpts{Delete: true}) + require.NoError(t, err) + sort.Slice(res, func(i, j int) bool { return res[i] < res[j] }) + assert.Equal(t, []string{"d1/file11.txt", "d1/file12.txt", "d2/file21.txt", "d2/file22.txt", "file1.txt", "file2.txt", "file3.txt"}, res) + + t.Run("delete dir with excluded files", func(t *testing.T) { + err = sess.Delete(ctx, "/tmp/delete.dest", &DeleteOpts{Recursive: true, Exclude: []string{"file2.*", "d1/*", "d2/file21.txt"}}) + assert.NoError(t, err) + out, e := sess.Run(ctx, "ls -1 /tmp/", &RunOpts{Verbose: true}) + require.NoError(t, e) + assert.Contains(t, out, "delete.dest", out) + + out, e = sess.Run(ctx, "ls -1 /tmp/delete.dest", &RunOpts{Verbose: true}) + require.NoError(t, e) + assert.Contains(t, out, "d1", out) + assert.Contains(t, out, "d2", out) + assert.Contains(t, out, "file2.txt", out) + assert.NotContains(t, out, "file3.txt", out) + + out, e = sess.Run(ctx, "ls -1 /tmp/delete.dest/d1", &RunOpts{Verbose: true}) + require.NoError(t, e) + assert.Contains(t, out, "file12.txt", out) + + out, e = sess.Run(ctx, "ls -1 /tmp/delete.dest/d2", &RunOpts{Verbose: true}) + require.NoError(t, e) + assert.Contains(t, out, "file21.txt", out) + assert.NotContains(t, out, "file22.txt", out) + }) +} + func TestRemote_CloseNoSession(t *testing.T) { sess := &Remote{} err := sess.Close() diff --git a/pkg/executor/testdata/data3.txt b/pkg/executor/testdata/data3.txt new file mode 100644 index 00000000..e3d45004 --- /dev/null +++ b/pkg/executor/testdata/data3.txt @@ -0,0 +1 @@ +data3 content \ No newline at end of file diff --git a/pkg/executor/testdata/delete/d1/file11.txt b/pkg/executor/testdata/delete/d1/file11.txt new file mode 100644 index 00000000..e5db5a5e --- /dev/null +++ b/pkg/executor/testdata/delete/d1/file11.txt @@ -0,0 +1 @@ +this is something \ No newline at end of file diff --git a/pkg/executor/testdata/delete/d1/file12.txt b/pkg/executor/testdata/delete/d1/file12.txt new file mode 100644 index 00000000..bc1a8be7 --- /dev/null +++ b/pkg/executor/testdata/delete/d1/file12.txt @@ -0,0 +1 @@ +this is something else \ No newline at end of file diff --git a/pkg/executor/testdata/delete/d2/file21.txt b/pkg/executor/testdata/delete/d2/file21.txt new file mode 100644 index 00000000..e5db5a5e --- /dev/null +++ b/pkg/executor/testdata/delete/d2/file21.txt @@ -0,0 +1 @@ +this is something \ No newline at end of file diff --git a/pkg/executor/testdata/delete/d2/file22.txt b/pkg/executor/testdata/delete/d2/file22.txt new file mode 100644 index 00000000..e5db5a5e --- /dev/null +++ b/pkg/executor/testdata/delete/d2/file22.txt @@ -0,0 +1 @@ +this is something \ No newline at end of file diff --git a/pkg/executor/testdata/delete/file1.txt b/pkg/executor/testdata/delete/file1.txt new file mode 100644 index 00000000..4a1eb1a0 --- /dev/null +++ b/pkg/executor/testdata/delete/file1.txt @@ -0,0 +1,6 @@ +// Process is a struct that holds the information needed to run a process. +type Process struct { + RemoteHosts []string + Concurrency int + Executors []remoteExecuter // executors pool +} \ No newline at end of file diff --git a/pkg/executor/testdata/delete/file2.txt b/pkg/executor/testdata/delete/file2.txt new file mode 100644 index 00000000..e294f5af --- /dev/null +++ b/pkg/executor/testdata/delete/file2.txt @@ -0,0 +1,3 @@ +Some random text to use for tests +blah blah blah +123 456 789 diff --git a/pkg/executor/testdata/delete/file3.txt b/pkg/executor/testdata/delete/file3.txt new file mode 100644 index 00000000..381360d2 --- /dev/null +++ b/pkg/executor/testdata/delete/file3.txt @@ -0,0 +1,2 @@ +Some random text to use for tests +Lorem ipsum dolor sit amet, consectetur adipiscing elit diff --git a/pkg/runner/commands.go b/pkg/runner/commands.go index 64afcf00..2a42ccd0 100644 --- a/pkg/runner/commands.go +++ b/pkg/runner/commands.go @@ -106,7 +106,7 @@ func (ec *execCmd) Copy(ctx context.Context) (resp execCmdResp, err error) { if !ec.cmd.Options.Sudo { // if sudo is not set, we can use the original destination and upload the file directly resp.details = fmt.Sprintf(" {copy: %s -> %s}", src, dst) - opts := &executor.UpDownOpts{Mkdir: ec.cmd.Copy.Mkdir, Force: ec.cmd.Copy.Force} + opts := &executor.UpDownOpts{Mkdir: ec.cmd.Copy.Mkdir, Force: ec.cmd.Copy.Force, Exclude: ec.cmd.Copy.Exclude} if err := ec.exec.Upload(ctx, src, dst, opts); err != nil { return resp, fmt.Errorf("can't copy file to %s: %w", ec.hostAddr, err) } @@ -117,7 +117,7 @@ func (ec *execCmd) Copy(ctx context.Context) (resp execCmdResp, err error) { // if sudo is set, we need to upload the file to a temporary directory and move it to the final destination resp.details = fmt.Sprintf(" {copy: %s -> %s, sudo: true}", src, dst) tmpDest := filepath.Join(tmpRemoteDir, filepath.Base(dst)) - if err := ec.exec.Upload(ctx, src, tmpDest, &executor.UpDownOpts{Mkdir: true, Force: true}); err != nil { + if err := ec.exec.Upload(ctx, src, tmpDest, &executor.UpDownOpts{Mkdir: true, Force: true, Exclude: ec.cmd.Copy.Exclude}); err != nil { // upload to a temporary directory with mkdir return resp, fmt.Errorf("can't copy file to %s: %w", ec.hostAddr, err) }