implement support of passing all exported vars to other commands

README.md

@@ -312,6 +312,23 @@ echo "All done! $FOO $BAR"
 
 By using this approach, Spot enables users to write and execute more complex scripts, providing greater flexibility and power in managing remote hosts or local environments.
 
+### Passing variables from one script command to another
+
+Spot allows passing variables from one script command to another. This is useful when you need to pass the output of one command to another command. For example if one command creates a file and you need to pass the file name to another command. To pass such variables, user need to use usual shell's `export` command in the first script command, and then all the variables exported in the first command will be available in the subsequent commands. 
+
+For example:
+
+```yaml
+commands:
+  - name: first command
+    script: |
+      export FILE_NAME=/tmp/file1
+      touch $FILE_NAME
+  - name: second command
+    script: |
+      echo "File name is $FILE_NAME"
+```
+
 ## Targets
 
 Targets are used to define the remote hosts to execute the tasks on. Targets can be defined in the playbook file or passed as a command-line argument. The following target types are supported:

pkg/config/command.go

@@ -65,7 +65,7 @@ type WaitInternal struct {
 }
 
 // GetScript returns a script string and an io.Reader based on the command being single line or multiline.
-func (cmd *Cmd) GetScript() (string, io.Reader) {
+func (cmd *Cmd) GetScript() (command string, rdr io.Reader) {
 	if cmd.Script == "" {
 		return "", nil
 	}
@@ -118,7 +118,7 @@ func (cmd *Cmd) scriptCommand(inp string) string {
 
 // GetScriptFile returns a reader for script file. All the line in the command used as a script, with hashbang,
 // set -e and environment variables.
-func (cmd *Cmd) scriptFile(inp string) io.Reader {
+func (cmd *Cmd) scriptFile(inp string) (r io.Reader) {
 	var buf bytes.Buffer
 
 	buf.WriteString("#!/bin/sh\n") // add hashbang
@@ -133,6 +133,7 @@ func (cmd *Cmd) scriptFile(inp string) io.Reader {
 		}
 	}
 
+	exports := []string{}
 	elems := strings.Split(inp, "\n")
 	for _, el := range elems {
 		c := strings.TrimSpace(el)
@@ -147,6 +148,22 @@ func (cmd *Cmd) scriptFile(inp string) io.Reader {
 		}
 		buf.WriteString(c)
 		buf.WriteString("\n")
+
+		// if the line in the script is an export, add it to the list of exports
+		// this is done to be able to print the variables set by the script to the console after the script is executed
+		// those variables can be used by the caller to set environment variables for the next commands
+		if strings.HasPrefix(c, "export") {
+			expKey := strings.TrimPrefix(c, "export")
+			expKey = strings.Split(expKey, "=")[0]
+			expKey = strings.TrimSpace(expKey)
+			exports = append(exports, expKey)
+		}
+	}
+
+	if len(exports) > 0 {
+		for i := range exports {
+			buf.WriteString(fmt.Sprintf("echo setvar %s=${%s}\n", exports[i], exports[i]))
+		}
 	}
 
 	return &buf

pkg/config/command_test.go

@@ -40,6 +40,27 @@ echo 'Goodbye, World!'`,
 				"echo 'Goodbye, World!'",
 			},
 		},
+		{
+			name: "multiline command with exports",
+			cmd: &Cmd{
+				Script: `echo 'Hello, World!'
+export FOO='bar'
+echo 'Goodbye, World!'
+export BAR='foo'
+`,
+			},
+			expectedScript: "",
+			expectedContents: []string{
+				"#!/bin/sh",
+				"set -e",
+				"echo 'Hello, World!'",
+				"export FOO='bar'",
+				"echo 'Goodbye, World!'",
+				"export BAR='foo'",
+				"echo setvar FOO=${FOO}",
+				"echo setvar BAR=${BAR}",
+			},
+		},
 		{
 			name: "single line command with environment variables",
 			cmd: &Cmd{
@@ -101,7 +122,6 @@ echo 'Goodbye, World!'`,
 		t.Run(tc.name, func(t *testing.T) {
 			script, reader := tc.cmd.GetScript()
 			assert.Equal(t, tc.expectedScript, script)
-
 			if reader != nil {
 				contents, err := io.ReadAll(reader)
 				assert.NoError(t, err)

pkg/runner/runner.go

@@ -159,7 +159,7 @@ func (p *Process) runTaskOnHost(ctx context.Context, tsk *config.Task, hostAddr,
 			}
 		}
 
-		details, err := p.execCommand(ctx, params)
+		details, vars, err := p.execCommand(ctx, params)
 		if err != nil {
 			if !cmd.Options.IgnoreErrors {
 				return count, fmt.Errorf("failed command %q on host %s (%s): %w", cmd.Name, hostAddr, hostName, err)
@@ -170,6 +170,25 @@ func (p *Process) runTaskOnHost(ctx context.Context, tsk *config.Task, hostAddr,
 			continue
 		}
 
+		// set variables from command output, if any
+		// this variables will be available for next commands in the same task via environment
+		if len(vars) > 0 {
+			log.Printf("[DEBUG] set %d variables from command %q: %+v", len(vars), cmd.Name, vars)
+			for k, v := range vars {
+				for i, c := range tsk.Commands {
+					env := c.Environment
+					if env == nil {
+						env = make(map[string]string)
+					}
+					if _, ok := env[k]; ok { // don't allow override variables
+						continue
+					}
+					env[k] = v
+					tsk.Commands[i].Environment = env
+				}
+			}
+		}
+
 		fmt.Fprintf(p.ColorWriter.WithHost(hostAddr, hostName),
 			"completed command %q%s (%v)", cmd.Name, details, time.Since(stCmd).Truncate(time.Millisecond))
 		count++
@@ -191,7 +210,7 @@ type execCmdParams struct {
 
 // execCommand executes a single command on a target host. It detects command type based on the fields what are set.
 // Even if multiple fields for multiple commands are set, only one will be executed.
-func (p *Process) execCommand(ctx context.Context, ep execCmdParams) (details string, err error) {
+func (p *Process) execCommand(ctx context.Context, ep execCmdParams) (details string, vars map[string]string, err error) {
 	switch {
 	case ep.cmd.Script != "":
 		log.Printf("[DEBUG] execute script %q on %s", ep.cmd.Name, ep.hostAddr)
@@ -212,24 +231,25 @@ func (p *Process) execCommand(ctx context.Context, ep execCmdParams) (details st
 		log.Printf("[DEBUG] wait for command on %s", ep.hostAddr)
 		return p.execWaitCommand(ctx, ep)
 	default:
-		return "", fmt.Errorf("unknown command %q", ep.cmd.Name)
+		return "", nil, fmt.Errorf("unknown command %q", ep.cmd.Name)
 	}
 }
 
 // execScriptCommand executes a script command on a target host. It can be a single line or multiline script,
 // this part is translated by the prepScript function.
 // If sudo option is set, it will execute the script with sudo.
-func (p *Process) execScriptCommand(ctx context.Context, ep execCmdParams) (details string, err error) {
+// If output contains variables as "setvar foo=bar", it will return the variables as map.
+func (p *Process) execScriptCommand(ctx context.Context, ep execCmdParams) (details string, vars map[string]string, err error) {
 	single, multiRdr := ep.cmd.GetScript()
 	c, teardown, err := p.prepScript(ctx, single, multiRdr, ep)
 	if err != nil {
-		return details, fmt.Errorf("can't prepare script on %s: %w", ep.hostAddr, err)
+		return details, nil, fmt.Errorf("can't prepare script on %s: %w", ep.hostAddr, err)
 	}
 	defer func() {
 		if teardown == nil {
 			return
 		}
-		if err := teardown(); err != nil {
+		if err = teardown(); err != nil {
 			log.Printf("[WARN] can't teardown script on %s: %v", ep.hostAddr, err)
 		}
 	}()
@@ -238,16 +258,31 @@ func (p *Process) execScriptCommand(ctx context.Context, ep execCmdParams) (deta
 		details = fmt.Sprintf(" {script: %s, sudo: true}", c)
 		c = fmt.Sprintf("sudo sh -c %q", c)
 	}
-	if _, err := ep.exec.Run(ctx, c, p.Verbose); err != nil {
-		return details, fmt.Errorf("can't run script on %s: %w", ep.hostAddr, err)
+	out, err := ep.exec.Run(ctx, c, p.Verbose)
+	if err != nil {
+		return details, nil, fmt.Errorf("can't run script on %s: %w", ep.hostAddr, err)
+	}
+
+	// collect setenv output and set it to the environment. This is needed for the next commands.
+	vars = make(map[string]string)
+	for _, line := range out {
+		if !strings.HasPrefix(line, "setvar ") {
+			continue
+		}
+		parts := strings.SplitN(strings.TrimPrefix(line, "setvar"), "=", 2)
+		if len(parts) != 2 {
+			continue
+		}
+		vars[strings.TrimSpace(parts[0])] = strings.TrimSpace(parts[1])
 	}
-	return details, nil
+
+	return details, vars, nil
 }
 
 // execCopyCommand upload a single file or multiple files (if wildcard is used) to a target host.
 // if sudo option is set, it will make a temporary directory and upload the files there,
 // then move it to the final destination with sudo script execution.
-func (p *Process) execCopyCommand(ctx context.Context, ep execCmdParams) (details string, err error) {
+func (p *Process) execCopyCommand(ctx context.Context, ep execCmdParams) (details string, vars map[string]string, err error) {
 	src := p.applyTemplates(ep.cmd.Copy.Source,
 		templateData{hostAddr: ep.hostAddr, hostName: ep.hostName, task: ep.tsk, command: ep.cmd.Name})
 	dst := p.applyTemplates(ep.cmd.Copy.Dest,
@@ -257,17 +292,17 @@ func (p *Process) execCopyCommand(ctx context.Context, ep execCmdParams) (detail
 		// if sudo is not set, we can use the original destination and upload the file directly
 		details = fmt.Sprintf(" {copy: %s -> %s}", src, dst)
 		if err := ep.exec.Upload(ctx, src, dst, ep.cmd.Copy.Mkdir); err != nil {
-			return details, fmt.Errorf("can't copy file to %s: %w", ep.hostAddr, err)
+			return details, nil, fmt.Errorf("can't copy file to %s: %w", ep.hostAddr, err)
 		}
-		return details, nil
+		return details, nil, nil
 	}
 
 	if ep.cmd.Options.Sudo {
 		// if sudo is set, we need to upload the file to a temporary directory and move it to the final destination
 		details = fmt.Sprintf(" {copy: %s -> %s, sudo: true}", src, dst)
 		tmpDest := filepath.Join(tmpRemoteDir, filepath.Base(dst))
 		if err := ep.exec.Upload(ctx, src, tmpDest, true); err != nil { // upload to a temporary directory with mkdir
-			return details, fmt.Errorf("can't copy file to %s: %w", ep.hostAddr, err)
+			return details, nil, fmt.Errorf("can't copy file to %s: %w", ep.hostAddr, err)
 		}
 
 		mvCmd := fmt.Sprintf("mv -f %s %s", tmpDest, dst) // move a single file
@@ -282,19 +317,19 @@ func (p *Process) execCopyCommand(ctx context.Context, ep execCmdParams) (detail
 		}
 		c, _, err := p.prepScript(ctx, mvCmd, nil, ep)
 		if err != nil {
-			return details, fmt.Errorf("can't prepare sudo moving command on %s: %w", ep.hostAddr, err)
+			return details, nil, fmt.Errorf("can't prepare sudo moving command on %s: %w", ep.hostAddr, err)
 		}
 
 		sudoMove := fmt.Sprintf("sudo %s", c)
 		if _, err := ep.exec.Run(ctx, sudoMove, p.Verbose); err != nil {
-			return details, fmt.Errorf("can't move file to %s: %w", ep.hostAddr, err)
+			return details, nil, fmt.Errorf("can't move file to %s: %w", ep.hostAddr, err)
 		}
 	}
 
-	return details, nil
+	return details, nil, nil
 }
 
-func (p *Process) execMCopyCommand(ctx context.Context, ep execCmdParams) (details string, err error) {
+func (p *Process) execMCopyCommand(ctx context.Context, ep execCmdParams) (details string, vars map[string]string, err error) {
 	msgs := []string{}
 	for _, c := range ep.cmd.MCopy {
 		src := p.applyTemplates(c.Source,
@@ -304,34 +339,34 @@ func (p *Process) execMCopyCommand(ctx context.Context, ep execCmdParams) (detai
 		msgs = append(msgs, fmt.Sprintf("%s -> %s", src, dst))
 		epSingle := ep
 		epSingle.cmd.Copy = config.CopyInternal{Source: src, Dest: dst, Mkdir: c.Mkdir}
-		if _, err := p.execCopyCommand(ctx, epSingle); err != nil {
-			return details, fmt.Errorf("can't copy file to %s: %w", ep.hostAddr, err)
+		if _, _, err := p.execCopyCommand(ctx, epSingle); err != nil {
+			return details, nil, fmt.Errorf("can't copy file to %s: %w", ep.hostAddr, err)
 		}
 	}
 	details = fmt.Sprintf(" {copy: %s}", strings.Join(msgs, ", "))
-	return details, nil
+	return details, nil, nil
 }
 
-func (p *Process) execSyncCommand(ctx context.Context, ep execCmdParams) (details string, err error) {
+func (p *Process) execSyncCommand(ctx context.Context, ep execCmdParams) (details string, vars map[string]string, err error) {
 	src := p.applyTemplates(ep.cmd.Sync.Source,
 		templateData{hostAddr: ep.hostAddr, hostName: ep.hostName, task: ep.tsk, command: ep.cmd.Name})
 	dst := p.applyTemplates(ep.cmd.Sync.Dest,
 		templateData{hostAddr: ep.hostAddr, hostName: ep.hostName, task: ep.tsk, command: ep.cmd.Name})
 	details = fmt.Sprintf(" {sync: %s -> %s}", src, dst)
 	if _, err := ep.exec.Sync(ctx, src, dst, ep.cmd.Sync.Delete); err != nil {
-		return details, fmt.Errorf("can't sync files on %s: %w", ep.hostAddr, err)
+		return details, nil, fmt.Errorf("can't sync files on %s: %w", ep.hostAddr, err)
 	}
-	return details, nil
+	return details, nil, nil
 }
 
-func (p *Process) execDeleteCommand(ctx context.Context, ep execCmdParams) (details string, err error) {
+func (p *Process) execDeleteCommand(ctx context.Context, ep execCmdParams) (details string, vars map[string]string, err error) {
 	loc := p.applyTemplates(ep.cmd.Delete.Location,
 		templateData{hostAddr: ep.hostAddr, hostName: ep.hostName, task: ep.tsk, command: ep.cmd.Name})
 
 	if !ep.cmd.Options.Sudo {
 		// if sudo is not set, we can delete the file directly
 		if err := ep.exec.Delete(ctx, loc, ep.cmd.Delete.Recursive); err != nil {
-			return details, fmt.Errorf("can't delete files on %s: %w", ep.hostAddr, err)
+			return details, nil, fmt.Errorf("can't delete files on %s: %w", ep.hostAddr, err)
 		}
 		details = fmt.Sprintf(" {delete: %s, recursive: %v}", loc, ep.cmd.Delete.Recursive)
 	}
@@ -343,17 +378,17 @@ func (p *Process) execDeleteCommand(ctx context.Context, ep execCmdParams) (deta
 			cmd = fmt.Sprintf("sudo rm -rf %s", loc)
 		}
 		if _, err := ep.exec.Run(ctx, cmd, p.Verbose); err != nil {
-			return details, fmt.Errorf("can't delete file(s) on %s: %w", ep.hostAddr, err)
+			return details, nil, fmt.Errorf("can't delete file(s) on %s: %w", ep.hostAddr, err)
 		}
 		details = fmt.Sprintf(" {delete: %s, recursive: %v, sudo: true}", loc, ep.cmd.Delete.Recursive)
 	}
 
-	return details, nil
+	return details, nil, nil
 }
 
 // execWaitCommand waits for a command to complete on a target hostAddr. It runs the command in a loop with a check duration
 // until the command succeeds or the timeout is exceeded.
-func (p *Process) execWaitCommand(ctx context.Context, ep execCmdParams) (details string, err error) {
+func (p *Process) execWaitCommand(ctx context.Context, ep execCmdParams) (details string, vars map[string]string, err error) {
 	c := p.applyTemplates(ep.cmd.Wait.Command,
 		templateData{hostAddr: ep.hostAddr, hostName: ep.hostName, task: ep.tsk, command: ep.cmd.Name})
 
@@ -383,12 +418,12 @@ func (p *Process) execWaitCommand(ctx context.Context, ep execCmdParams) (detail
 	for {
 		select {
 		case <-ctx.Done():
-			return details, ctx.Err()
+			return details, nil, ctx.Err()
 		case <-timeoutTk.C:
-			return details, fmt.Errorf("timeout exceeded")
+			return details, nil, fmt.Errorf("timeout exceeded")
 		case <-checkTk.C:
 			if _, err := ep.exec.Run(ctx, waitCmd, false); err == nil {
-				return details, nil // command succeeded
+				return details, nil, nil // command succeeded
 			}
 		}
 	}