add support for task-level options propagated to all commands

the initial goal was to allow setting task-level secrets, however, to keep things consistent with other options it was extended to support all the options. The solution has a limitation and command options can't reset boolean options defined on task level. This is because we don't have a way to distinguish between default values (i.e. false for bool) and false set for real.

README.md

@@ -452,6 +452,17 @@ example setting `ignore_errors`, `no_auto` and `only_on` options:
         options: {ignore_errors: true, no_auto: true, only_on: [host1, host2]}
 ```
 
+The same options can be set for the whole task as well. In this case, the options will be applied to all commands in the task but can be overridden for a specific command. Pls note: the command option cannot reset the boolean options that were set for the task. This limitation is due to the way the default values are set.
+
+```yaml
+  - name: deploy-things
+    on_error: "curl -s localhost:8080/error?msg={SPOT_ERROR}" # call hook on error
+    options: {ignore_errors: true, no_auto: true, only_on: [host1, host2]}
+    commands:
+      - name: wait
+        script: sleep 5s
+```
+
 ### Command conditionals
 
 `cond`: defines a condition for the command to be executed. The condition is a valid shell command that will be executed on the remote host(s) and if it returns 0, the primary command will be executed. For example, `cond: "test -f /tmp/foo"` will execute the primary script command only if the file `/tmp/foo` exists. The condition can be reversed by adding `!` prefix, i.e. `! test -f /tmp/foo` will pass only if the file `/tmp/foo` doesn't exist.
@@ -825,6 +836,22 @@ tasks:
 
 In this case secrets for keys `user`, `password` and `token` will be read from the secrets provider, decrypted at runtime, and passed to the command in the environment. Please note: if a user runs `spot` with the `--verbose` or `--dbg` flag, the secrets will be replaced with `****` in the output. This is done to prevent secrets from being displayed or logged.
 
+
+Sometimes, users may want to use the same set of secrets in multiple commands. To avoid repeating the secrets in each command, users can set `secrets` at the task level, as shown in the following example:
+
+```yaml
+tasks:
+  - name: access sensitive data
+    commands:
+      - name: read api response
+        script: |
+          curl -s -u ${user}:${password} https://api.example.com  
+          curl https://api.example.com -H "Authorization: Bearer ${token}"
+    options:
+      secrets: [user, password, token]
+```
+
+
 ### Built-in Secrets Provider
 
 Spot includes a built-in secrets provider that can be used to store secrets in SQLite, MySQL, or Postgresql databases. The provider can be configured using the following command line options or environment variables:

pkg/config/playbook.go

@@ -46,23 +46,25 @@ type SecretsProvider interface {
 // SimplePlayBook defines simplified top-level config
 // It is used for unmarshalling only, and result used to make the usual PlayBook
 type SimplePlayBook struct {
-	User       string   `yaml:"user" toml:"user"`               // ssh user
-	SSHKey     string   `yaml:"ssh_key" toml:"ssh_key"`         // ssh key
-	SSHShell   string   `yaml:"ssh_shell" toml:"ssh_shell"`     // ssh shell to uses
-	LocalShell string   `yaml:"local_shell" toml:"local_shell"` // local shell to use
-	Inventory  string   `yaml:"inventory" toml:"inventory"`     // inventory file or url
-	Targets    []string `yaml:"targets" toml:"targets"`         // list of names
-	Target     string   `yaml:"target" toml:"target"`           // a single target to run task on
-	Task       []Cmd    `yaml:"task" toml:"task"`               // single task is a list of commands
+	User       string     `yaml:"user" toml:"user"`                 // ssh user
+	SSHKey     string     `yaml:"ssh_key" toml:"ssh_key"`           // ssh key
+	SSHShell   string     `yaml:"ssh_shell" toml:"ssh_shell"`       // ssh shell to uses
+	LocalShell string     `yaml:"local_shell" toml:"local_shell"`   // local shell to use
+	Inventory  string     `yaml:"inventory" toml:"inventory"`       // inventory file or url
+	Targets    []string   `yaml:"targets" toml:"targets"`           // list of names
+	Target     string     `yaml:"target" toml:"target"`             // a single target to run task on
+	Task       []Cmd      `yaml:"task" toml:"task"`                 // single task is a list of commands
+	Options    CmdOptions `yaml:"options" toml:"options,omitempty"` // options for all commands
 }
 
 // Task defines multiple commands runs together
 type Task struct {
-	Name     string   `yaml:"name" toml:"name"` // name of task, mandatory
-	User     string   `yaml:"user" toml:"user"`
-	Commands []Cmd    `yaml:"commands" toml:"commands"`
-	OnError  string   `yaml:"on_error" toml:"on_error"`
-	Targets  []string `yaml:"targets" toml:"targets"` // optional list of targets to run task on, names or groups
+	Name     string     `yaml:"name" toml:"name"` // name of task, mandatory
+	User     string     `yaml:"user" toml:"user"`
+	Commands []Cmd      `yaml:"commands" toml:"commands"`
+	OnError  string     `yaml:"on_error" toml:"on_error"`
+	Targets  []string   `yaml:"targets" toml:"targets"`           // optional list of targets to run task on, names or groups
+	Options  CmdOptions `yaml:"options" toml:"options,omitempty"` // options for all commands
 }
 
 // Target defines hosts to run commands on
@@ -148,22 +150,42 @@ func New(fname string, overrides *Overrides, secProvider SecretsProvider) (res *
 		return nil, fmt.Errorf("config %s is invalid: %w", fname, err)
 	}
 
-	// load secrets from secrets provider
-	if secErr := res.loadSecrets(); secErr != nil {
-		return nil, secErr
-	}
-
-	// log loaded config info
 	log.Printf("[INFO] playbook loaded with %d tasks", len(res.Tasks))
+
 	for i, tsk := range res.Tasks {
 		for j, c := range tsk.Commands {
-			// set shell (remote and local) for all commands in task
+			// set shell (remote and local) for all commands in the task
 			res.Tasks[i].Commands[j].SSHShell = res.remoteShell()
 			res.Tasks[i].Commands[j].LocalShell = res.localShell()
+
+			// append task's secret keys to all the commands
+			res.Tasks[i].Commands[j].Options.Secrets = append(res.Tasks[i].Commands[j].Options.Secrets, tsk.Options.Secrets...)
+			// append task's only_on to all the commands
+			res.Tasks[i].Commands[j].Options.OnlyOn = append(res.Tasks[i].Commands[j].Options.OnlyOn, tsk.Options.OnlyOn...)
+
+			// set bool options for all commands in the task, but only if they are set in the task to true to avoid overriding
+			if tsk.Options.Local {
+				res.Tasks[i].Commands[j].Options.Local = tsk.Options.Local
+			}
+			if tsk.Options.NoAuto {
+				res.Tasks[i].Commands[j].Options.NoAuto = tsk.Options.NoAuto
+			}
+			if tsk.Options.IgnoreErrors {
+				res.Tasks[i].Commands[j].Options.IgnoreErrors = tsk.Options.IgnoreErrors
+			}
+			if tsk.Options.Sudo {
+				res.Tasks[i].Commands[j].Options.Sudo = tsk.Options.Sudo
+			}
+
 			log.Printf("[DEBUG] load command %q (task: %s)", c.Name, tsk.Name)
 		}
 	}
 
+	// load secrets from secrets provider
+	if secErr := res.loadSecrets(); secErr != nil {
+		return nil, secErr
+	}
+
 	// load inventory if set
 	inventoryLoc := os.Getenv(inventoryEnv) // default inventory location from env
 	if res.Inventory != "" {

pkg/config/playbook_test.go

@@ -169,6 +169,10 @@ func TestPlaybook_New(t *testing.T) {
 					return "VAL1", nil
 				case "SEC2":
 					return "VAL2", nil
+				case "SEC11":
+					return "VAL11", nil
+				case "SEC12":
+					return "VAL12", nil
 				default:
 					return "", fmt.Errorf("unknown secret key %q", key)
 				}
@@ -181,14 +185,59 @@ func TestPlaybook_New(t *testing.T) {
 		assert.Equal(t, "deploy-remark42", p.Tasks[0].Name, "task name")
 		assert.Equal(t, 5, len(p.Tasks[0].Commands), "5 commands")
 
-		assert.Equal(t, map[string]string{"SEC1": "VAL1", "SEC2": "VAL2"}, p.secrets, "Secrets map for all Secrets")
+		assert.Equal(t, map[string]string{"SEC1": "VAL1", "SEC11": "VAL11", "SEC12": "VAL12", "SEC2": "VAL2"},
+			p.secrets, "Secrets map for all Secrets")
 
 		tsk, err := p.Task("deploy-remark42")
 		require.NoError(t, err)
 		assert.Equal(t, 5, len(tsk.Commands))
 		assert.Equal(t, "docker", tsk.Commands[4].Name)
-		assert.Equal(t, map[string]string{"SEC1": "VAL1", "SEC2": "VAL2"}, tsk.Commands[4].Secrets)
-		assert.Equal(t, []string{"VAL1", "VAL2"}, p.AllSecretValues())
+		assert.Equal(t, map[string]string{"SEC1": "VAL1", "SEC11": "VAL11", "SEC12": "VAL12", "SEC2": "VAL2"}, tsk.Commands[4].Secrets)
+		assert.Equal(t, []string{"VAL1", "VAL11", "VAL12", "VAL2"}, p.AllSecretValues())
+	})
+
+	t.Run("playbook with options", func(t *testing.T) {
+		secProvider := &mocks.SecretProvider{
+			GetFunc: func(key string) (string, error) {
+				switch key {
+				case "SEC1":
+					return "VAL1", nil
+				case "SEC2":
+					return "VAL2", nil
+				case "SEC11":
+					return "VAL11", nil
+				case "SEC12":
+					return "VAL12", nil
+				default:
+					return "", fmt.Errorf("unknown secret key %q", key)
+				}
+			},
+		}
+
+		p, err := New("testdata/playbook-with-task-opts.yml", nil, secProvider)
+		require.NoError(t, err)
+		assert.Equal(t, 1, len(p.Tasks), "1 task")
+		assert.Equal(t, "deploy-remark42", p.Tasks[0].Name, "task name")
+		assert.Equal(t, 5, len(p.Tasks[0].Commands), "5 commands")
+
+		assert.Equal(t, map[string]string{"SEC1": "VAL1", "SEC11": "VAL11", "SEC12": "VAL12", "SEC2": "VAL2"},
+			p.secrets, "Secrets map for all Secrets")
+
+		tsk, err := p.Task("deploy-remark42")
+		require.NoError(t, err)
+		assert.Equal(t, 5, len(tsk.Commands))
+		assert.Equal(t, "docker", tsk.Commands[4].Name)
+		assert.EqualValues(t, map[string]string{"SEC1": "VAL1", "SEC11": "VAL11", "SEC12": "VAL12", "SEC2": "VAL2"}, tsk.Commands[4].Secrets)
+		assert.Equal(t, []string{"VAL1", "VAL11", "VAL12", "VAL2"}, p.AllSecretValues())
+
+		assert.Equal(t, CmdOptions{IgnoreErrors: true, NoAuto: true, Secrets: []string{"SEC11", "SEC12"}}, p.Tasks[0].Commands[0].Options)
+		assert.Equal(t, CmdOptions{IgnoreErrors: true, NoAuto: true, Secrets: []string{"SEC11", "SEC12"}}, p.Tasks[0].Commands[1].Options)
+		assert.Equal(t, CmdOptions{IgnoreErrors: true, NoAuto: true, Local: true,
+			Secrets: []string{"SEC11", "SEC12"}}, p.Tasks[0].Commands[2].Options)
+		assert.Equal(t, CmdOptions{IgnoreErrors: true, NoAuto: true, Local: false, Sudo: true,
+			Secrets: []string{"SEC11", "SEC12"}}, p.Tasks[0].Commands[3].Options)
+		assert.Equal(t, CmdOptions{IgnoreErrors: true, NoAuto: true, Local: false, Sudo: false,
+			Secrets: []string{"SEC1", "SEC2", "SEC11", "SEC12"}}, p.Tasks[0].Commands[4].Options)
 	})
 
 	t.Run("playbook prohibited all target", func(t *testing.T) {

pkg/config/testdata/playbook-with-secrets.yml

@@ -7,6 +7,8 @@ targets:
 
 tasks:
   - name: deploy-remark42
+    options:
+        secrets: ["SEC11", "SEC12"]
     commands:
       - name: wait
         script: sleep 5

pkg/config/testdata/playbook-with-task-opts.yml

@@ -0,0 +1,46 @@
+user: umputun
+
+targets:
+    remark42:
+      hosts: [{name: "h1", host: "h1.example.com"}, {host: "h2.example.com"}]
+
+
+tasks:
+  - name: deploy-remark42
+    options:
+        secrets: ["SEC11", "SEC12"]
+        no_auto: true
+        ignore_errors: true
+    commands:
+      - name: wait
+        script: sleep 5
+
+      - name: copy configuration
+        copy: {"src": "/local/remark42.yml", "dst": "/srv/remark42.yml", "mkdir": true}
+
+      - name: some local command
+        options: {local: true}
+        script: |
+          ls -la /srv
+          du -hcs /srv
+
+      - name: git
+        options: {no_auto: true, sudo: true}
+        before: "echo before git"
+        after: "echo after git"
+        onerror: "echo onerror git"
+        script: |
+          git clone https://example.com/remark42.git /srv || true # clone if doesn't exists, but don't fail if exists
+          cd /srv
+          git pull
+
+      - name: docker
+        options: {no_auto: true, secrets: ["SEC1", "SEC2"]}
+        script: |
+          docker pull umputun/remark42:latest
+          docker stop remark42 || true
+          docker rm remark42 || true
+          docker run -d --name remark42 -p 8080:8080 umputun/remark42:latest
+        env:
+          FOO: bar
+          BAR: qux