From c4db01cafac034e3e6ded5d00a452cf12ec01293 Mon Sep 17 00:00:00 2001 From: Umputun Date: Fri, 22 Mar 2024 18:48:52 -0500 Subject: [PATCH] add support for task-level options propagated to all commands the initial goal was to allow setting task-level secrets, however, to keep things consistent with other options it was extended to support all the options. The solution has a limitation and command options can't reset boolean options defined on task level. This is because we don't have a way to distinguish between default values (i.e. false for bool) and false set for real. --- README.md | 27 ++++++++ pkg/config/playbook.go | 62 +++++++++++++------ pkg/config/playbook_test.go | 55 +++++++++++++++- pkg/config/testdata/playbook-with-secrets.yml | 2 + .../testdata/playbook-with-task-opts.yml | 46 ++++++++++++++ 5 files changed, 169 insertions(+), 23 deletions(-) create mode 100644 pkg/config/testdata/playbook-with-task-opts.yml diff --git a/README.md b/README.md index 937c0dcf..a077d06c 100644 --- a/README.md +++ b/README.md @@ -452,6 +452,17 @@ example setting `ignore_errors`, `no_auto` and `only_on` options: options: {ignore_errors: true, no_auto: true, only_on: [host1, host2]} ``` +The same options can be set for the whole task as well. In this case, the options will be applied to all commands in the task but can be overridden for a specific command. Pls note: the command option cannot reset the boolean options that were set for the task. This limitation is due to the way the default values are set. + +```yaml + - name: deploy-things + on_error: "curl -s localhost:8080/error?msg={SPOT_ERROR}" # call hook on error + options: {ignore_errors: true, no_auto: true, only_on: [host1, host2]} + commands: + - name: wait + script: sleep 5s +``` + ### Command conditionals `cond`: defines a condition for the command to be executed. The condition is a valid shell command that will be executed on the remote host(s) and if it returns 0, the primary command will be executed. For example, `cond: "test -f /tmp/foo"` will execute the primary script command only if the file `/tmp/foo` exists. The condition can be reversed by adding `!` prefix, i.e. `! test -f /tmp/foo` will pass only if the file `/tmp/foo` doesn't exist. @@ -825,6 +836,22 @@ tasks: In this case secrets for keys `user`, `password` and `token` will be read from the secrets provider, decrypted at runtime, and passed to the command in the environment. Please note: if a user runs `spot` with the `--verbose` or `--dbg` flag, the secrets will be replaced with `****` in the output. This is done to prevent secrets from being displayed or logged. + +Sometimes, users may want to use the same set of secrets in multiple commands. To avoid repeating the secrets in each command, users can set `secrets` at the task level, as shown in the following example: + +```yaml +tasks: + - name: access sensitive data + commands: + - name: read api response + script: | + curl -s -u ${user}:${password} https://api.example.com + curl https://api.example.com -H "Authorization: Bearer ${token}" + options: + secrets: [user, password, token] +``` + + ### Built-in Secrets Provider Spot includes a built-in secrets provider that can be used to store secrets in SQLite, MySQL, or Postgresql databases. The provider can be configured using the following command line options or environment variables: diff --git a/pkg/config/playbook.go b/pkg/config/playbook.go index 25607eaa..4117030f 100644 --- a/pkg/config/playbook.go +++ b/pkg/config/playbook.go @@ -46,23 +46,25 @@ type SecretsProvider interface { // SimplePlayBook defines simplified top-level config // It is used for unmarshalling only, and result used to make the usual PlayBook type SimplePlayBook struct { - User string `yaml:"user" toml:"user"` // ssh user - SSHKey string `yaml:"ssh_key" toml:"ssh_key"` // ssh key - SSHShell string `yaml:"ssh_shell" toml:"ssh_shell"` // ssh shell to uses - LocalShell string `yaml:"local_shell" toml:"local_shell"` // local shell to use - Inventory string `yaml:"inventory" toml:"inventory"` // inventory file or url - Targets []string `yaml:"targets" toml:"targets"` // list of names - Target string `yaml:"target" toml:"target"` // a single target to run task on - Task []Cmd `yaml:"task" toml:"task"` // single task is a list of commands + User string `yaml:"user" toml:"user"` // ssh user + SSHKey string `yaml:"ssh_key" toml:"ssh_key"` // ssh key + SSHShell string `yaml:"ssh_shell" toml:"ssh_shell"` // ssh shell to uses + LocalShell string `yaml:"local_shell" toml:"local_shell"` // local shell to use + Inventory string `yaml:"inventory" toml:"inventory"` // inventory file or url + Targets []string `yaml:"targets" toml:"targets"` // list of names + Target string `yaml:"target" toml:"target"` // a single target to run task on + Task []Cmd `yaml:"task" toml:"task"` // single task is a list of commands + Options CmdOptions `yaml:"options" toml:"options,omitempty"` // options for all commands } // Task defines multiple commands runs together type Task struct { - Name string `yaml:"name" toml:"name"` // name of task, mandatory - User string `yaml:"user" toml:"user"` - Commands []Cmd `yaml:"commands" toml:"commands"` - OnError string `yaml:"on_error" toml:"on_error"` - Targets []string `yaml:"targets" toml:"targets"` // optional list of targets to run task on, names or groups + Name string `yaml:"name" toml:"name"` // name of task, mandatory + User string `yaml:"user" toml:"user"` + Commands []Cmd `yaml:"commands" toml:"commands"` + OnError string `yaml:"on_error" toml:"on_error"` + Targets []string `yaml:"targets" toml:"targets"` // optional list of targets to run task on, names or groups + Options CmdOptions `yaml:"options" toml:"options,omitempty"` // options for all commands } // Target defines hosts to run commands on @@ -148,22 +150,42 @@ func New(fname string, overrides *Overrides, secProvider SecretsProvider) (res * return nil, fmt.Errorf("config %s is invalid: %w", fname, err) } - // load secrets from secrets provider - if secErr := res.loadSecrets(); secErr != nil { - return nil, secErr - } - - // log loaded config info log.Printf("[INFO] playbook loaded with %d tasks", len(res.Tasks)) + for i, tsk := range res.Tasks { for j, c := range tsk.Commands { - // set shell (remote and local) for all commands in task + // set shell (remote and local) for all commands in the task res.Tasks[i].Commands[j].SSHShell = res.remoteShell() res.Tasks[i].Commands[j].LocalShell = res.localShell() + + // append task's secret keys to all the commands + res.Tasks[i].Commands[j].Options.Secrets = append(res.Tasks[i].Commands[j].Options.Secrets, tsk.Options.Secrets...) + // append task's only_on to all the commands + res.Tasks[i].Commands[j].Options.OnlyOn = append(res.Tasks[i].Commands[j].Options.OnlyOn, tsk.Options.OnlyOn...) + + // set bool options for all commands in the task, but only if they are set in the task to true to avoid overriding + if tsk.Options.Local { + res.Tasks[i].Commands[j].Options.Local = tsk.Options.Local + } + if tsk.Options.NoAuto { + res.Tasks[i].Commands[j].Options.NoAuto = tsk.Options.NoAuto + } + if tsk.Options.IgnoreErrors { + res.Tasks[i].Commands[j].Options.IgnoreErrors = tsk.Options.IgnoreErrors + } + if tsk.Options.Sudo { + res.Tasks[i].Commands[j].Options.Sudo = tsk.Options.Sudo + } + log.Printf("[DEBUG] load command %q (task: %s)", c.Name, tsk.Name) } } + // load secrets from secrets provider + if secErr := res.loadSecrets(); secErr != nil { + return nil, secErr + } + // load inventory if set inventoryLoc := os.Getenv(inventoryEnv) // default inventory location from env if res.Inventory != "" { diff --git a/pkg/config/playbook_test.go b/pkg/config/playbook_test.go index a7bdaaf7..ef49efb9 100644 --- a/pkg/config/playbook_test.go +++ b/pkg/config/playbook_test.go @@ -169,6 +169,10 @@ func TestPlaybook_New(t *testing.T) { return "VAL1", nil case "SEC2": return "VAL2", nil + case "SEC11": + return "VAL11", nil + case "SEC12": + return "VAL12", nil default: return "", fmt.Errorf("unknown secret key %q", key) } @@ -181,14 +185,59 @@ func TestPlaybook_New(t *testing.T) { assert.Equal(t, "deploy-remark42", p.Tasks[0].Name, "task name") assert.Equal(t, 5, len(p.Tasks[0].Commands), "5 commands") - assert.Equal(t, map[string]string{"SEC1": "VAL1", "SEC2": "VAL2"}, p.secrets, "Secrets map for all Secrets") + assert.Equal(t, map[string]string{"SEC1": "VAL1", "SEC11": "VAL11", "SEC12": "VAL12", "SEC2": "VAL2"}, + p.secrets, "Secrets map for all Secrets") tsk, err := p.Task("deploy-remark42") require.NoError(t, err) assert.Equal(t, 5, len(tsk.Commands)) assert.Equal(t, "docker", tsk.Commands[4].Name) - assert.Equal(t, map[string]string{"SEC1": "VAL1", "SEC2": "VAL2"}, tsk.Commands[4].Secrets) - assert.Equal(t, []string{"VAL1", "VAL2"}, p.AllSecretValues()) + assert.Equal(t, map[string]string{"SEC1": "VAL1", "SEC11": "VAL11", "SEC12": "VAL12", "SEC2": "VAL2"}, tsk.Commands[4].Secrets) + assert.Equal(t, []string{"VAL1", "VAL11", "VAL12", "VAL2"}, p.AllSecretValues()) + }) + + t.Run("playbook with options", func(t *testing.T) { + secProvider := &mocks.SecretProvider{ + GetFunc: func(key string) (string, error) { + switch key { + case "SEC1": + return "VAL1", nil + case "SEC2": + return "VAL2", nil + case "SEC11": + return "VAL11", nil + case "SEC12": + return "VAL12", nil + default: + return "", fmt.Errorf("unknown secret key %q", key) + } + }, + } + + p, err := New("testdata/playbook-with-task-opts.yml", nil, secProvider) + require.NoError(t, err) + assert.Equal(t, 1, len(p.Tasks), "1 task") + assert.Equal(t, "deploy-remark42", p.Tasks[0].Name, "task name") + assert.Equal(t, 5, len(p.Tasks[0].Commands), "5 commands") + + assert.Equal(t, map[string]string{"SEC1": "VAL1", "SEC11": "VAL11", "SEC12": "VAL12", "SEC2": "VAL2"}, + p.secrets, "Secrets map for all Secrets") + + tsk, err := p.Task("deploy-remark42") + require.NoError(t, err) + assert.Equal(t, 5, len(tsk.Commands)) + assert.Equal(t, "docker", tsk.Commands[4].Name) + assert.EqualValues(t, map[string]string{"SEC1": "VAL1", "SEC11": "VAL11", "SEC12": "VAL12", "SEC2": "VAL2"}, tsk.Commands[4].Secrets) + assert.Equal(t, []string{"VAL1", "VAL11", "VAL12", "VAL2"}, p.AllSecretValues()) + + assert.Equal(t, CmdOptions{IgnoreErrors: true, NoAuto: true, Secrets: []string{"SEC11", "SEC12"}}, p.Tasks[0].Commands[0].Options) + assert.Equal(t, CmdOptions{IgnoreErrors: true, NoAuto: true, Secrets: []string{"SEC11", "SEC12"}}, p.Tasks[0].Commands[1].Options) + assert.Equal(t, CmdOptions{IgnoreErrors: true, NoAuto: true, Local: true, + Secrets: []string{"SEC11", "SEC12"}}, p.Tasks[0].Commands[2].Options) + assert.Equal(t, CmdOptions{IgnoreErrors: true, NoAuto: true, Local: false, Sudo: true, + Secrets: []string{"SEC11", "SEC12"}}, p.Tasks[0].Commands[3].Options) + assert.Equal(t, CmdOptions{IgnoreErrors: true, NoAuto: true, Local: false, Sudo: false, + Secrets: []string{"SEC1", "SEC2", "SEC11", "SEC12"}}, p.Tasks[0].Commands[4].Options) }) t.Run("playbook prohibited all target", func(t *testing.T) { diff --git a/pkg/config/testdata/playbook-with-secrets.yml b/pkg/config/testdata/playbook-with-secrets.yml index 04bfa04a..8729d862 100644 --- a/pkg/config/testdata/playbook-with-secrets.yml +++ b/pkg/config/testdata/playbook-with-secrets.yml @@ -7,6 +7,8 @@ targets: tasks: - name: deploy-remark42 + options: + secrets: ["SEC11", "SEC12"] commands: - name: wait script: sleep 5 diff --git a/pkg/config/testdata/playbook-with-task-opts.yml b/pkg/config/testdata/playbook-with-task-opts.yml new file mode 100644 index 00000000..e91de725 --- /dev/null +++ b/pkg/config/testdata/playbook-with-task-opts.yml @@ -0,0 +1,46 @@ +user: umputun + +targets: + remark42: + hosts: [{name: "h1", host: "h1.example.com"}, {host: "h2.example.com"}] + + +tasks: + - name: deploy-remark42 + options: + secrets: ["SEC11", "SEC12"] + no_auto: true + ignore_errors: true + commands: + - name: wait + script: sleep 5 + + - name: copy configuration + copy: {"src": "/local/remark42.yml", "dst": "/srv/remark42.yml", "mkdir": true} + + - name: some local command + options: {local: true} + script: | + ls -la /srv + du -hcs /srv + + - name: git + options: {no_auto: true, sudo: true} + before: "echo before git" + after: "echo after git" + onerror: "echo onerror git" + script: | + git clone https://example.com/remark42.git /srv || true # clone if doesn't exists, but don't fail if exists + cd /srv + git pull + + - name: docker + options: {no_auto: true, secrets: ["SEC1", "SEC2"]} + script: | + docker pull umputun/remark42:latest + docker stop remark42 || true + docker rm remark42 || true + docker run -d --name remark42 -p 8080:8080 umputun/remark42:latest + env: + FOO: bar + BAR: qux \ No newline at end of file