add Exclude support for Delete and Copy/Mcopy commands

README.md

@@ -333,7 +333,7 @@ script: |
 Copies a file from the local machine to the remote host(s). If `mkdir` is set to `true` the command will create the destination directory if it doesn't exist, same as `mkdir -p` in bash. The command also supports glob patterns in `src` field.
 
 Copy command performs a quick check to see if the file already exists on the remote host(s) with the same size and modification time,
-and skips the copy if it does. This option can be disabled by setting `force: true` flag.
+and skips the copy if it does. This option can be disabled by setting `force: true` flag. Another option is `exclude` which allows to specify a list of files to exclude to be copied.
 
 ```yaml
 - name: copy file with mkdir
@@ -342,6 +342,8 @@ and skips the copy if it does. This option can be disabled by setting `force: tr
 - name: copy files with glob
   copy: {"src": "testdata/*.csv", "dst": "/tmp/things"}
 
+- name: copy files with glob and exclude
+  copy: {"src": "testdata/*.yml", "dst": "/tmp/things", "exclude": ["conf.dist.yml"]}
 
 - name: copy files with force flag
   copy: {"src": "testdata/*.csv", "dst": "/tmp/things", "force": true}
@@ -360,7 +362,6 @@ Copy also supports list format to copy multiple files at once:
 
 Synchronises directory from the local machine to the remote host(s). Optionally supports deleting files on the remote host(s) that don't exist locally with `"delete": true` flag. Another option is `exclude` which allows to specify a list of files to exclude from the sync.
 
-
 ```yaml
 - name: sync directory
   sync: {"src": "testdata", "dst": "/tmp/things"}
@@ -382,8 +383,12 @@ Deletes a file or directory on the remote host(s), optionally can remove recursi
 ```yaml
 - name: delete file
   delete: {"path": "/tmp/things.csv"}
+
 - name: delete directory recursively
   delete: {"path": "/tmp/things", "recur": true}
+
+- name: delete directory recursively with exclude
+  delete: {"path": "/tmp/things", "recur": true, "exclude": ["*.txt", "*.yml"]}
 ```
 
 Delete also supports list format to remove multiple paths at once.

pkg/config/command.go

@@ -44,10 +44,11 @@ type CmdOptions struct {
 
 // CopyInternal defines copy command, implemented internally
 type CopyInternal struct {
-	Source string `yaml:"src" toml:"src"`     // source must be a file or a glob pattern
-	Dest   string `yaml:"dst" toml:"dst"`     // destination must be a file or a directory
-	Mkdir  bool   `yaml:"mkdir" toml:"mkdir"` // create destination directory if it does not exist
-	Force  bool   `yaml:"force" toml:"force"` // force copy even if source and destination are the same
+	Source  string   `yaml:"src" toml:"src"`         // source must be a file or a glob pattern
+	Dest    string   `yaml:"dst" toml:"dst"`         // destination must be a file or a directory
+	Mkdir   bool     `yaml:"mkdir" toml:"mkdir"`     // create destination directory if it does not exist
+	Force   bool     `yaml:"force" toml:"force"`     // force copy even if source and destination are the same
+	Exclude []string `yaml:"exclude" toml:"exclude"` // exclude files matching these patterns
 }
 
 // SyncInternal defines sync command (recursive copy), implemented internally
@@ -61,8 +62,9 @@ type SyncInternal struct {
 
 // DeleteInternal defines delete command, implemented internally
 type DeleteInternal struct {
-	Location  string `yaml:"path" toml:"path"`
-	Recursive bool   `yaml:"recur" toml:"recur"`
+	Location  string   `yaml:"path" toml:"path"`
+	Recursive bool     `yaml:"recur" toml:"recur"`
+	Exclude   []string `yaml:"exclude" toml:"exclude"`
 }
 
 // WaitInternal defines wait command, implemented internally

pkg/executor/dry.go

@@ -45,8 +45,15 @@ func (ex *Dry) Run(_ context.Context, cmd string, opts *RunOpts) (out []string,
 
 // Upload doesn't actually upload, just prints the command
 func (ex *Dry) Upload(_ context.Context, local, remote string, opts *UpDownOpts) (err error) {
-	mkdir := opts != nil && opts.Mkdir
-	log.Printf("[DEBUG] upload %s to %s, mkdir: %v", local, remote, mkdir)
+	var mkdir bool
+	var exclude []string
+
+	if opts != nil {
+		mkdir = opts.Mkdir
+		exclude = opts.Exclude
+	}
+
+	log.Printf("[DEBUG] upload %s to %s, mkdir: %v, exclude: %v", local, remote, mkdir, exclude)
 	if strings.Contains(remote, "spot-script") {
 		// this is a temp script created by spot to perform script execution on remote host
 		outLog, outErr := MakeOutAndErrWriters(ex.hostAddr, ex.hostName, true, ex.secrets)
@@ -71,8 +78,15 @@ func (ex *Dry) Upload(_ context.Context, local, remote string, opts *UpDownOpts)
 
 // Download file from remote server with scp
 func (ex *Dry) Download(_ context.Context, remote, local string, opts *UpDownOpts) (err error) {
-	mkdir := opts != nil && opts.Mkdir
-	log.Printf("[DEBUG] download %s to %s, mkdir: %v", local, remote, mkdir)
+	var mkdir bool
+	var exclude []string
+
+	if opts != nil {
+		mkdir = opts.Mkdir
+		exclude = opts.Exclude
+	}
+
+	log.Printf("[DEBUG] download %s to %s, mkdir: %v, exclude: %v", local, remote, mkdir, exclude)
 	return nil
 }
 
@@ -89,8 +103,14 @@ func (ex *Dry) Sync(_ context.Context, localDir, remoteDir string, opts *SyncOpt
 
 // Delete doesn't delete anything, just prints the command
 func (ex *Dry) Delete(_ context.Context, remoteFile string, opts *DeleteOpts) (err error) {
-	recursive := opts != nil && opts.Recursive
-	log.Printf("[DEBUG] delete %s, recursive: %v", remoteFile, recursive)
+	var recursive bool
+	var exclude []string
+
+	if opts != nil {
+		recursive = opts.Recursive
+		exclude = opts.Exclude
+	}
+	log.Printf("[DEBUG] delete %s, recursive: %v, exclude: %v", remoteFile, recursive, exclude)
 	return nil
 }
 

pkg/executor/executor.go

@@ -37,9 +37,10 @@ type RunOpts struct {
 
 // UpDownOpts is a struct for upload and download options.
 type UpDownOpts struct {
-	Mkdir    bool // create remote directory if it does not exist
-	Checksum bool // compare checksums of local and remote files, default is size and modtime
-	Force    bool // overwrite existing files on remote
+	Mkdir    bool     // create remote directory if it does not exist
+	Checksum bool     // compare checksums of local and remote files, default is size and modtime
+	Force    bool     // overwrite existing files on remote
+	Exclude  []string // exclude files matching the given patterns
 }
 
 // SyncOpts is a struct for sync options.
@@ -52,7 +53,8 @@ type SyncOpts struct {
 
 // DeleteOpts is a struct for delete options.
 type DeleteOpts struct {
-	Recursive bool // delete directories recursively
+	Recursive bool     // delete directories recursively
+	Exclude   []string // exclude files matching the given patterns
 }
 
 // StdOutLogWriter is a writer that writes to log with a prefix and a log level.
@@ -184,6 +186,20 @@ func isExcluded(path string, excl []string) bool {
 	return false
 }
 
+func isExcludedSubPath(path string, excl []string) bool {
+	subpath := filepath.Join(path, "*")
+	for _, ex := range excl {
+		match, err := filepath.Match(subpath, strings.TrimSuffix(ex, "/*"))
+		if err != nil {
+			continue
+		}
+		if match {
+			return true
+		}
+	}
+	return false
+}
+
 func isWithinOneSecond(t1, t2 time.Time) bool {
 	diff := t1.Sub(t2)
 	if diff < 0 {

pkg/executor/local.go

@@ -60,13 +60,29 @@ func (l *Local) Upload(_ context.Context, src, dst string, opts *UpDownOpts) (er
 		return fmt.Errorf("source file %q not found", src)
 	}
 
-	if opts != nil && opts.Mkdir {
+	var mkdir bool
+	var exclude []string
+
+	if opts != nil {
+		mkdir = opts.Mkdir
+		exclude = opts.Exclude
+	}
+
+	if mkdir {
 		if err = os.MkdirAll(filepath.Dir(dst), 0o750); err != nil {
 			return fmt.Errorf("can't create local dir %s: %w", filepath.Dir(dst), err)
 		}
 	}
 
 	for _, match := range matches {
+		relPath, e := filepath.Rel(filepath.Dir(src), match)
+		if e != nil {
+			return fmt.Errorf("failed to build relative path for %s: %w", match, err)
+		}
+		if isExcluded(relPath, exclude) {
+			continue
+		}
+
 		destination := dst
 		if len(matches) > 1 {
 			destination = filepath.Join(dst, filepath.Base(match))
@@ -128,12 +144,18 @@ func (l *Local) Sync(ctx context.Context, src, dst string, opts *SyncOpts) ([]st
 }
 
 // Delete file or directory
-func (l *Local) Delete(_ context.Context, remoteFile string, opts *DeleteOpts) (err error) {
+func (l *Local) Delete(ctx context.Context, remoteFile string, opts *DeleteOpts) (err error) {
 	recursive := opts != nil && opts.Recursive
 	if !recursive {
 		return os.Remove(remoteFile)
 	}
-	return os.RemoveAll(remoteFile)
+
+	var exclude []string
+	if opts != nil {
+		exclude = opts.Exclude
+	}
+
+	return l.deletePath(ctx, remoteFile, exclude)
 }
 
 // Close does nothing for local
@@ -259,3 +281,65 @@ func (l *Local) copyFile(src, dst string) error {
 
 	return nil
 }
+
+func (l *Local) deletePath(ctx context.Context, src string, excl []string) error {
+	info, err := os.Stat(src)
+	if err != nil {
+		return err
+	}
+
+	if !info.IsDir() || len(excl) == 0 {
+		return os.RemoveAll(src)
+	}
+
+	hasExclusion := false
+	err = filepath.Walk(src, func(filePath string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+
+		if ctx.Err() != nil {
+			return ctx.Err()
+		}
+
+		relPath, err := filepath.Rel(src, filePath)
+		if err != nil {
+			return err
+		}
+
+		if info.IsDir() && isExcludedSubPath(relPath, excl) {
+			return nil
+		}
+
+		if isExcluded(relPath, excl) {
+			hasExclusion = true
+			if info.IsDir() {
+				return filepath.SkipDir
+			}
+
+			return nil
+		}
+
+		if !info.IsDir() {
+			return os.Remove(filePath)
+		}
+
+		err = os.RemoveAll(filePath)
+		if err != nil {
+			return err
+		}
+
+		return filepath.SkipDir
+	})
+
+	if err != nil {
+		return err
+	}
+
+	// remove the whole directory if there are no actual exclusions
+	if !hasExclusion {
+		return os.RemoveAll(src)
+	}
+
+	return nil
+}