Dynamic targets

README.md

@@ -284,12 +284,37 @@ Each task consists of a list of commands that will be executed on the remote hos
 
 - `on_error`: specifies the command to execute on the local host (the one running the `spot` command) in case of an error. The command can use the `{SPOT_ERROR}` variable to access the last error message. Example: `on_error: "curl -s localhost:8080/error?msg={SPOT_ERROR}"`
 - `user`: specifies the SSH user to use when connecting to remote hosts. Overrides the user defined in the top section of playbook file for the specified task.
-- `targets` - list of target names, group, tags or host addresses to execute the task on. Command line `-t` flag can be used to override this field. 
+- `targets` - list of target names, group, tags or host addresses to execute the task on. Command line `-t` flag can be used to override this field. The `targets` field may include variables. For more details see [Dynamic targets](#dynamic-targets) section.
 
 *Note: these fields supported in the full playbook type only*
 
 All tasks are executed sequentially one a given host, one after another. If a task fails, the execution of the playbook will stop and the `on_error` command will be executed on the local host, if defined. Every task has to have `name` field defined, which is used to identify the task everywhere. Playbook with missing `name` field will fail to execute immediately. Duplicate task names are not allowed either.
 
+## Dynamic targets
+
+Spot offers support for dynamic targets, allowing the list of targets to be defined dynamically using variables. This feature becomes particularly useful when users need to ascertain a destination address within one task, and subsequently use it in another task. Here is an illustrative example:
+
+```yaml
+tasks:
+  - name: get host
+    targets: ["default"]
+    script: |
+      export thehost=$(curl -s http://example.com/next-host)
+    options: {local: true}
+
+  - name: run on host
+    targets: ["$thehost"]
+    script: |
+      echo "doing something on $thehost"
+```
+
+In this example, the host address is initially fetched from http://example.com/next-host. Following this, the task "run on host" is executed on the host that was just identified. This ability to use dynamic targets proves beneficial in a variety of scenarios, especially when the list of hosts is not predetermined.
+
+A practical use case for dynamic targets arises during the provisioning of a new host, followed by the execution of commands on it. Since the IP address of the new host isn't known beforehand, dynamic retrieval becomes essential.
+
+_The reason the first task specifies `targets: ["default"]` is because Spot requires some target to execute a task. In this case, all commands in "get host" tasks are local and won't be invoked on a remote host. The `default` target is utilized by Spot if no alternative target is specified via the command line._
+
+
 ## Relative paths resolution
 
 Relative path resolution is a frequent issue in systems that involve file references or inclusion. Different systems handle this in various ways. Spot uses a widely-adopted method of resolving relative paths based on the current working directory of the process. This means that if you run Spot from different directories, the way relative paths are resolved will change. In simpler terms, Spot doesn't resolve relative paths according to the location of the playbook file itself.

cmd/spot/main.go

@@ -178,7 +178,7 @@ func run(opts options) error {
 	if opts.PositionalArgs.AdHocCmd != "" { // run ad-hoc command
 		r.Verbose = true // always verbose for ad-hoc
 		for _, targetName := range opts.Targets {
-			if err := runTaskForTarget(ctx, r, "ad-hoc", targetName); err != nil {
+			if err := runTaskForTarget(ctx, r, "ad-hoc", targetName, conf); err != nil {
 				errs = multierror.Append(errs, err)
 			}
 		}
@@ -187,7 +187,7 @@ func run(opts options) error {
 
 	if opts.TaskName != "" { // run a single task
 		for _, targetName := range targetsForTask(opts, opts.TaskName, conf) {
-			if err := runTaskForTarget(ctx, r, opts.TaskName, targetName); err != nil {
+			if err := runTaskForTarget(ctx, r, opts.TaskName, targetName, conf); err != nil {
 				return err
 			}
 		}
@@ -197,7 +197,7 @@ func run(opts options) error {
 	// run all tasks
 	for _, task := range conf.Tasks {
 		for _, targetName := range targetsForTask(opts, task.Name, conf) {
-			if err := runTaskForTarget(ctx, r, task.Name, targetName); err != nil {
+			if err := runTaskForTarget(ctx, r, task.Name, targetName, conf); err != nil {
 				return err
 			}
 		}
@@ -206,14 +206,15 @@ func run(opts options) error {
 	return nil
 }
 
-func runTaskForTarget(ctx context.Context, r runner.Process, taskName, targetName string) error {
+func runTaskForTarget(ctx context.Context, r runner.Process, taskName, targetName string, conf *config.PlayBook) error {
 	st := time.Now()
-	stats, err := r.Run(ctx, taskName, targetName)
+	res, err := r.Run(ctx, taskName, targetName)
 	if err != nil {
 		return fmt.Errorf("can't run task %q for target %q: %w", taskName, targetName, err)
 	}
 	log.Printf("[INFO] completed: hosts:%d, commands:%d in %v\n",
-		stats.Hosts, stats.Commands, time.Since(st).Truncate(100*time.Millisecond))
+		res.Hosts, res.Commands, time.Since(st).Truncate(100*time.Millisecond))
+	conf.UpdateTasksTargets(res.Vars)
 	return nil
 }
 

cmd/spot/main_test.go

@@ -102,6 +102,25 @@ func Test_runCompleted(t *testing.T) {
 		require.NoError(t, err)
 		assert.True(t, time.Since(st) < 1*time.Second)
 	})
+
+	t.Run("run with dynamic targets", func(t *testing.T) {
+		opts := options{
+			SSHUser:      "test",
+			SSHKey:       "testdata/test_ssh_key",
+			PlaybookFile: "testdata/conf-dynamic.yml",
+			SecretsProvider: SecretsProvider{
+				Provider: "spot",
+				Conn:     "testdata/test-secrets.db",
+				Key:      "1234567890",
+			},
+			Env: map[string]string{
+				"hostAndPort": hostAndPort,
+			},
+		}
+		setupLog(true)
+		err := run(opts)
+		require.NoError(t, err)
+	})
 }
 
 func Test_runCompletedSimplePlaybook(t *testing.T) {

cmd/spot/testdata/conf-dynamic.yml

@@ -0,0 +1,33 @@
+targets:
+  remark42:
+    hosts: [{host: "h1.example.com"}, {host: "h2.example.com"}]
+  staging:
+    inventory_file: {location: "testdata/inventory"}
+
+
+tasks:
+  - name: task1
+    targets: ["default"]
+    commands:
+      - name: some command
+        script: |
+          export host2=$hostAndPort
+          ls -laR /tmp
+          echo "blah" > /tmp/conf.yml
+          cat /tmp/conf.yml
+          echo "all good, 123 - $FOO $BAR"
+        env:
+          FOO: foo-val
+          BAR: bar-val
+        options: {local: true}
+
+
+  - name: task2
+    targets: ["$host2"]
+    commands:
+      - name: good command
+        script: echo good command 1
+      - name: good command
+        script: echo good command 2
+      - name: task vars
+        script: env

pkg/config/playbook.go

@@ -363,6 +363,28 @@ func (p *PlayBook) AllSecretValues() []string {
 	return res
 }
 
+// UpdateTasksTargets updates the targets of all tasks in the playbook with the values from the specified map of variables.
+// The method is used to replace variables in the targets of tasks with their actual values and this way provide dynamic targets.
+func (p *PlayBook) UpdateTasksTargets(vars map[string]string) {
+	for i, task := range p.Tasks {
+		targets := []string{}
+		for _, tg := range task.Targets {
+			if len(tg) > 1 && strings.HasPrefix(tg, "$") {
+				if vars == nil {
+					continue
+				}
+				if v, ok := vars[tg[1:]]; ok {
+					log.Printf("[DEBUG] set target %s to %q", tg, v)
+					targets = append(targets, v)
+				}
+				continue
+			}
+			targets = append(targets, tg)
+		}
+		p.Tasks[i].Targets = targets
+	}
+}
+
 // loadInventory loads the inventory data from the specified location (file or URL) and returns it as an InventoryData struct.
 // The inventory data is parsed as either YAML or TOML, depending on the file extension.
 // The method also performs some additional processing on the inventory data:

pkg/config/playbook_test.go

@@ -368,6 +368,71 @@ func TestTargetHosts(t *testing.T) {
 	}
 }
 
+func TestPlayBook_UpdateTasksTargets(t *testing.T) {
+	tests := []struct {
+		name     string
+		playbook PlayBook
+		vars     map[string]string
+		expected PlayBook
+	}{
+		{
+			name: "replace target variables",
+			playbook: PlayBook{
+				Tasks: []Task{{Targets: []string{"$target1", "target2"}}, {Targets: []string{"target3", "$target4"}}},
+			},
+			vars: map[string]string{"target1": "actualTarget1", "target4": "actualTarget4"},
+			expected: PlayBook{
+				Tasks: []Task{{Targets: []string{"actualTarget1", "target2"}}, {Targets: []string{"target3", "actualTarget4"}}},
+			},
+		},
+		{
+			name: "ignore single dollar sign",
+			playbook: PlayBook{
+				Tasks: []Task{{Targets: []string{"$"}}},
+			},
+			vars: map[string]string{"$": "actualTarget1"},
+			expected: PlayBook{
+				Tasks: []Task{{Targets: []string{"$"}}},
+			},
+		},
+		{
+			name: "ignore undefined variables",
+			playbook: PlayBook{
+				Tasks: []Task{{Targets: []string{"$undefined"}}},
+			},
+			vars: map[string]string{},
+			expected: PlayBook{
+				Tasks: []Task{{Targets: []string{}}},
+			},
+		},
+		{
+			name:     "playbook with no tasks",
+			playbook: PlayBook{},
+			vars: map[string]string{
+				"target1": "actualTarget1",
+			},
+			expected: PlayBook{},
+		},
+		{
+			name: "nil target variables",
+			playbook: PlayBook{
+				Tasks: []Task{{Targets: []string{"$target1", "target2"}}, {Targets: []string{"target3", "$target4"}}},
+			},
+			vars: nil,
+			expected: PlayBook{
+				Tasks: []Task{{Targets: []string{"target2"}}, {Targets: []string{"target3"}}},
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			tt.playbook.UpdateTasksTargets(tt.vars)
+			assert.Equal(t, tt.expected, tt.playbook)
+		})
+	}
+}
+
 func TestPlayBook_loadInventory(t *testing.T) {
 	// create temporary inventory files
 	yamlData := []byte(`