From e71b32711fdb57238b6db1e40d658611ebb26fcf Mon Sep 17 00:00:00 2001 From: Umputun Date: Sun, 23 Jul 2023 14:38:33 -0500 Subject: [PATCH] add dbl-quotes to echo command nit: typo in assert message add a couple of tests add playbook-level `ssh_shell` fix empty shell in command if not set in env default local shell to /bin/sh if no SHELL var set add debug info for runner creation pin go version to 1.20.5 due to https://github.com/testcontainers/testcontainers-go/issues/1359 add support of --shell parameter setting the default ssh shell for the remote execution --- README.md | 5 +- cmd/spot/main.go | 8 ++- cmd/spot/main_test.go | 18 +++++- pkg/config/command.go | 25 +++++++-- pkg/config/command_test.go | 50 ++++++++++++++--- pkg/config/playbook.go | 22 ++++++-- pkg/config/playbook_test.go | 42 ++++++++++++++ pkg/config/testdata/with-ssh-shell.yml | 42 ++++++++++++++ pkg/executor/connector.go | 4 ++ pkg/executor/local.go | 16 +++++- pkg/executor/remote_test.go | 7 ++- pkg/runner/commands.go | 30 +++++++--- pkg/runner/commands_test.go | 4 +- pkg/runner/runner.go | 11 ++-- pkg/runner/runner_test.go | 3 +- spot.1 | 78 +++++++++++++++++++++++++- 16 files changed, 321 insertions(+), 44 deletions(-) create mode 100644 pkg/config/testdata/with-ssh-shell.yml diff --git a/README.md b/README.md index 793711cc..18b9f95d 100644 --- a/README.md +++ b/README.md @@ -134,6 +134,7 @@ Spot supports the following command-line options: - `-c`, `--concurrent=`: Sets the number of concurrent hosts to execute tasks. Defaults to `1`, which means hosts will be handled sequentially. - `--timeout`: Sets the SSH timeout. Defaults to `30s`. User can also set the environment variable `$SPOT_TIMEOUT` to define the SSH timeout. - `--ssh-agent`: Enables the use of the SSH agent for authentication. Defaults to `false`. User can also set the environment variable `SPOT_SSH_AGENT` to define the value. +- `--shell` - shell to use for remote ssh execution, default is `/bin/sh`. User can also set the environment variable `SPOT_SHELL` to define the value. - `-i`, `--inventory=`: Specifies the inventory file or url to use for the task execution. Overrides the inventory file defined in the playbook file. User can also set the environment variable `$SPOT_INVENTORY` to define the default inventory file path or url. - `-u`, `--user=`: Specifies the SSH user to use when connecting to remote hosts. Overrides the user defined in the playbook file . @@ -141,7 +142,7 @@ Spot supports the following command-line options: - `-s`, `--skip=`: Skips the specified commands during the task execution. Providing the `-s` flag multiple times with different command names skips multiple commands. - `-o`, `--only=`: Runs only the specified commands during the task execution. Providing the `-o` flag multiple times with different command names runs only multiple commands. - `-e`, `--env=`: Sets the environment variables to be used during the task execution. Providing the `-e` flag multiple times with different environment variables sets multiple environment variables, e.g., `-e VAR1:VALUE1 -e VAR2:VALUE2`. -- `-E`, `--env-file=`: Sets the environment variables from file to be used during the task execution. Default is env.yml` +- `-E`, `--env-file=`: Sets the environment variables from file to be used during the task execution. Default is env.yml. Can be also set with environment variable `SPOT_ENV_FILE`. - `--dry`: Enables dry-run mode, which prints out the commands to be executed without actually executing them. - `-v`, `--verbose`: Enables verbose mode, providing more detailed output and error messages during the task execution. - `--dbg`: Enables debug mode, providing even more detailed output and error messages during the task execution as well as diagnostic messages. @@ -166,6 +167,7 @@ Spot supports the following command-line options: ```yaml user: umputun # default ssh user. Can be overridden by -u flag or by inventory or host definition ssh_key: keys/id_rsa # ssh key +ssh_shell: /bin/bash # shell to use for remote ssh execution, default is /bin/sh inventory: /etc/spot/inventory.yml # default inventory file. Can be overridden by --inventory flag # list of targets, i.e. hosts, inventory files or inventory URLs @@ -235,6 +237,7 @@ In some cases the rich syntax of the full playbook is not needed and can felt ov ```yaml user: umputun # default ssh user. Can be overridden by -u flag or by inventory or host definition ssh_key: keys/id_rsa # ssh key +ssh_shell: /bin/bash # shell to use for remote ssh execution, default is /bin/sh inventory: /etc/spot/inventory.yml # default inventory file. Can be overridden by --inventory flag targets: ["devbox1", "devbox2", "h1.example.com:2222", "h2.example.com"] # list of host names from inventory and direct host ips diff --git a/cmd/spot/main.go b/cmd/spot/main.go index d663c623..6d05ef12 100644 --- a/cmd/spot/main.go +++ b/cmd/spot/main.go @@ -37,13 +37,14 @@ type options struct { Concurrent int `short:"c" long:"concurrent" description:"concurrent tasks" default:"1"` SSHTimeout time.Duration `long:"timeout" env:"SPOT_TIMEOUT" description:"ssh timeout" default:"30s"` SSHAgent bool `long:"ssh-agent" env:"SPOT_SSH_AGENT" description:"use ssh-agent"` + SSHShell string `long:"shell" env:"SPOT_SHELL" description:"shell to use for ssh" default:"/bin/sh"` // overrides Inventory string `short:"i" long:"inventory" description:"inventory file or url [$SPOT_INVENTORY]"` SSHUser string `short:"u" long:"user" description:"ssh user"` SSHKey string `short:"k" long:"key" description:"ssh key"` Env map[string]string `short:"e" long:"env" description:"environment variables for all commands"` - EnvFile string `short:"E" long:"env-file" description:"environment variables from file" default:"env.yml"` + EnvFile string `short:"E" long:"env-file" env:"SPOT_ENV_FILE" description:"environment variables from file" default:"env.yml"` // commands filter Skip []string `long:"skip" description:"skip commands"` @@ -287,6 +288,7 @@ func makePlaybook(opts options, inventory string) (*config.PlayBook, error) { Environment: env, User: opts.SSHUser, AdHocCommand: opts.PositionalArgs.AdHocCmd, + SSHShell: opts.SSHShell, } exPlaybookFile, err := expandPath(opts.PlaybookFile) @@ -334,7 +336,11 @@ func makeRunner(opts options, pbook *config.PlayBook) (*runner.Process, error) { ColorWriter: executor.NewColorizedWriter(os.Stdout, "", "", "", nil), Verbose: opts.Verbose, Dry: opts.Dry, + SSHShell: opts.SSHShell, } + log.Printf("[DEBUG] runner created: concurrency:%d, connector: %s, ssh_shell:%q, verbose:%v, dry:%v, only:%v, skip:%v", + r.Concurrency, r.Connector, r.SSHShell, r.Verbose, r.Dry, r.Only, r.Skip) + return &r, nil } diff --git a/cmd/spot/main_test.go b/cmd/spot/main_test.go index 5636a540..31f60eb5 100644 --- a/cmd/spot/main_test.go +++ b/cmd/spot/main_test.go @@ -26,9 +26,21 @@ func Test_main(t *testing.T) { hostAndPort, teardown := startTestContainer(t) defer teardown() - args := []string{"simplotask", "--dbg", "--playbook=testdata/conf-local.yml", "--user=test", "--key=testdata/test_ssh_key", "--target=" + hostAndPort} - os.Args = args - main() + t.Run("with system shell set", func(t *testing.T) { + args := []string{"simplotask", "--dbg", "--playbook=testdata/conf-local.yml", "--user=test", + "--key=testdata/test_ssh_key", "--target=" + hostAndPort} + os.Args = args + main() + }) + + t.Run("with system shell not set", func(t *testing.T) { + args := []string{"simplotask", "--dbg", "--playbook=testdata/conf-local.yml", "--user=test", + "--key=testdata/test_ssh_key", "--target=" + hostAndPort} + os.Args = args + err := os.Setenv("SHELL", "") + require.NoError(t, err) + main() + }) } func Test_runCompleted(t *testing.T) { diff --git a/pkg/config/command.go b/pkg/config/command.go index adc3df7c..2ae5be3d 100644 --- a/pkg/config/command.go +++ b/pkg/config/command.go @@ -5,6 +5,7 @@ import ( "fmt" "io" "log" + "os" "reflect" "sort" "strings" @@ -29,7 +30,8 @@ type Cmd struct { Options CmdOptions `yaml:"options" toml:"options,omitempty"` Condition string `yaml:"cond" toml:"cond,omitempty"` - Secrets map[string]string `yaml:"-" toml:"-"` // loaded secrets, filled by playbook + Secrets map[string]string `yaml:"-" toml:"-"` // loaded secrets, filled by playbook + SSHShell string `yaml:"-" toml:"-"` // shell to use for ssh commands, filled by playbook } // CmdOptions defines options for a command @@ -136,7 +138,7 @@ func (cmd *Cmd) scriptCommand(inp string) string { // add environment variables envs := cmd.genEnv() - res := "sh -c '" + res := cmd.shell() + " -c '" if len(envs) > 0 { res += strings.Join(envs, "; ") + "; " } @@ -169,8 +171,8 @@ func (cmd *Cmd) scriptFile(inp string) (r io.Reader) { var buf bytes.Buffer if !cmd.hasShebang(inp) { - buf.WriteString("#!/bin/sh\n") // add default shebang if not present - buf.WriteString("set -e\n") // add 'set -e' to make the script exit on error + buf.WriteString("#!" + cmd.shell() + "\n") // add default shebang if not present + buf.WriteString("set -e\n") // add 'set -e' to make the script exit on error } envs := cmd.genEnv() @@ -390,3 +392,18 @@ func (cmd *Cmd) validate() error { } return nil } + +func (cmd *Cmd) shell() string { + if cmd.SSHShell == "" { + return "/bin/sh" + } + if cmd.Options.Local { + envShell := os.Getenv("SHELL") + if envShell == "" { + return "/bin/sh" + } + return envShell + } + + return cmd.SSHShell +} diff --git a/pkg/config/command_test.go b/pkg/config/command_test.go index 6a19c09a..b98a3a26 100644 --- a/pkg/config/command_test.go +++ b/pkg/config/command_test.go @@ -23,7 +23,7 @@ func TestCmd_GetScript(t *testing.T) { cmd: &Cmd{ Script: "echo Hello, World!", }, - expectedScript: `sh -c 'echo Hello, World!'`, + expectedScript: `/bin/sh -c 'echo Hello, World!'`, expectedContents: nil, }, { @@ -93,7 +93,7 @@ export FOO='bar' "GREETING": "Hello, World!", }, }, - expectedScript: `sh -c 'GREETING="Hello, World!"; echo $GREETING'`, + expectedScript: `/bin/sh -c 'GREETING="Hello, World!"; echo $GREETING'`, expectedContents: nil, }, { @@ -181,7 +181,7 @@ func TestCmd_getScriptCommand(t *testing.T) { cmd := c.Tasks[0].Commands[3] assert.Equal(t, "git", cmd.Name, "name") res := cmd.scriptCommand(cmd.Script) - assert.Equal(t, `sh -c 'git clone https://example.com/remark42.git /srv || true; cd /srv; git pull'`, res) + assert.Equal(t, `/bin/sh -c 'git clone https://example.com/remark42.git /srv || true; cd /srv; git pull'`, res) }) t.Run("no-script", func(t *testing.T) { @@ -195,7 +195,35 @@ func TestCmd_getScriptCommand(t *testing.T) { cmd := c.Tasks[0].Commands[4] assert.Equal(t, "docker", cmd.Name) res := cmd.scriptCommand(cmd.Script) - assert.Equal(t, `sh -c 'BAR="qux"; FOO="bar"; docker pull umputun/remark42:latest; docker stop remark42 || true; docker rm remark42 || true; docker run -d --name remark42 -p 8080:8080 umputun/remark42:latest'`, res) + assert.Equal(t, `/bin/sh -c 'BAR="qux"; FOO="bar"; docker pull umputun/remark42:latest; docker stop remark42 || true; docker rm remark42 || true; docker run -d --name remark42 -p 8080:8080 umputun/remark42:latest'`, res) + }) +} + +func TestCmd_getScriptCommandCustomShell(t *testing.T) { + c, err := New("testdata/f1.yml", &Overrides{SSHShell: "/bin/bash"}, nil) + require.NoError(t, err) + t.Logf("%+v", c) + assert.Equal(t, 1, len(c.Tasks), "single task") + + t.Run("script", func(t *testing.T) { + cmd := c.Tasks[0].Commands[3] + assert.Equal(t, "git", cmd.Name, "name") + res := cmd.scriptCommand(cmd.Script) + assert.Equal(t, `/bin/bash -c 'git clone https://example.com/remark42.git /srv || true; cd /srv; git pull'`, res) + }) + + t.Run("no-script", func(t *testing.T) { + cmd := c.Tasks[0].Commands[1] + assert.Equal(t, "copy configuration", cmd.Name) + res := cmd.scriptCommand(cmd.Script) + assert.Equal(t, "", res) + }) + + t.Run("script with env", func(t *testing.T) { + cmd := c.Tasks[0].Commands[4] + assert.Equal(t, "docker", cmd.Name) + res := cmd.scriptCommand(cmd.Script) + assert.Equal(t, `/bin/bash -c 'BAR="qux"; FOO="bar"; docker pull umputun/remark42:latest; docker stop remark42 || true; docker rm remark42 || true; docker run -d --name remark42 -p 8080:8080 umputun/remark42:latest'`, res) }) } @@ -219,6 +247,14 @@ func TestCmd_getScriptFile(t *testing.T) { }, expected: "#!/bin/bash\nset -e\necho 'Hello, World!'\n", }, + { + name: "with non-default shell", + cmd: &Cmd{ + Script: "echo 'Hello, World!'", + SSHShell: "/bin/zsh", + }, + expected: "#!/bin/zsh\nset -e\necho 'Hello, World!'\n", + }, { name: "with one environment variable", cmd: &Cmd{ @@ -501,7 +537,7 @@ func TestCmd_GetWait(t *testing.T) { Command: "echo Hello, World!", }, }, - expectedCmd: `sh -c 'echo Hello, World!'`, + expectedCmd: `/bin/sh -c 'echo Hello, World!'`, }, { name: "multi-line wait command", @@ -551,13 +587,13 @@ func TestCmd_GetCondition(t *testing.T) { { name: "single-line wait command", cmd: &Cmd{Condition: "echo Hello, World!"}, - expectedCmd: `sh -c 'echo Hello, World!'`, + expectedCmd: `/bin/sh -c 'echo Hello, World!'`, expectedInvert: false, }, { name: "single-line wait command inverted", cmd: &Cmd{Condition: "! echo Hello, World!"}, - expectedCmd: `sh -c 'echo Hello, World!'`, + expectedCmd: `/bin/sh -c 'echo Hello, World!'`, expectedInvert: true, }, { diff --git a/pkg/config/playbook.go b/pkg/config/playbook.go index e44c4b2e..822232e5 100644 --- a/pkg/config/playbook.go +++ b/pkg/config/playbook.go @@ -26,6 +26,7 @@ import ( type PlayBook struct { User string `yaml:"user" toml:"user"` // ssh user SSHKey string `yaml:"ssh_key" toml:"ssh_key"` // ssh key + SSHShell string `yaml:"ssh_shell" toml:"ssh_shell"` // ssh shell to use Inventory string `yaml:"inventory" toml:"inventory"` // inventory file or url Targets map[string]Target `yaml:"targets" toml:"targets"` // list of targets/environments Tasks []Task `yaml:"tasks" toml:"tasks"` // list of tasks @@ -85,6 +86,7 @@ type Overrides struct { Inventory string Environment map[string]string AdHocCommand string + SSHShell string } // InventoryData defines inventory data format @@ -150,8 +152,9 @@ func New(fname string, overrides *Overrides, secProvider SecretsProvider) (res * // log loaded config info log.Printf("[INFO] playbook loaded with %d tasks", len(res.Tasks)) - for _, tsk := range res.Tasks { - for _, c := range tsk.Commands { + for i, tsk := range res.Tasks { + for j, c := range tsk.Commands { + res.Tasks[i].Commands[j].SSHShell = res.shell() // set secrets for all commands in task log.Printf("[DEBUG] load command %q (task: %s)", c.Name, tsk.Name) } } @@ -267,7 +270,7 @@ func unmarshalPlaybookFile(fname string, data []byte, overrides *Overrides, res } res.Targets = map[string]Target{"default": target} return nil - } else { //nolint + } else { // nolint errs = multierror.Append(errs, err) } @@ -296,7 +299,8 @@ func (p *PlayBook) Task(name string) (*Task, error) { searchTask := func(tsk []Task, name string) (*Task, error) { if name == "ad-hoc" && p.overrides.AdHocCommand != "" { // special case for ad-hoc command, make a fake task with a single command from overrides.AdHocCommand - return &Task{Name: "ad-hoc", Commands: []Cmd{{Name: "ad-hoc", Script: p.overrides.AdHocCommand}}}, nil + return &Task{Name: "ad-hoc", Commands: []Cmd{ + {Name: "ad-hoc", Script: p.overrides.AdHocCommand, SSHShell: p.shell()}}}, nil } for _, t := range tsk { if strings.EqualFold(t.Name, name) { @@ -585,3 +589,13 @@ func (p *PlayBook) loadSecrets() error { } return nil } + +func (p *PlayBook) shell() string { + if p.overrides != nil && p.overrides.SSHShell != "" { + return p.overrides.SSHShell + } + if p.SSHShell != "" { + return p.SSHShell + } + return "/bin/sh" +} diff --git a/pkg/config/playbook_test.go b/pkg/config/playbook_test.go index a95a6127..9545e4c6 100644 --- a/pkg/config/playbook_test.go +++ b/pkg/config/playbook_test.go @@ -22,6 +22,7 @@ func TestPlaybook_New(t *testing.T) { t.Logf("%+v", c) assert.Equal(t, 1, len(c.Tasks), "single task") assert.Equal(t, "umputun", c.User, "user") + assert.Equal(t, "/bin/sh", c.Tasks[0].Commands[0].SSHShell, "ssh shell") tsk := c.Tasks[0] assert.Equal(t, 5, len(tsk.Commands), "5 commands") @@ -182,12 +183,38 @@ func TestPlaybook_New(t *testing.T) { assert.Equal(t, 5, len(tsk.Commands)) assert.Equal(t, "docker", tsk.Commands[4].Name) assert.Equal(t, map[string]string{"SEC1": "VAL1", "SEC2": "VAL2"}, tsk.Commands[4].Secrets) + assert.Equal(t, []string{"VAL1", "VAL2"}, p.AllSecretValues()) }) t.Run("playbook prohibited all target", func(t *testing.T) { _, err := New("testdata/playbook-with-all-group.yml", nil, nil) require.ErrorContains(t, err, "config testdata/playbook-with-all-group.yml is invalid: target \"all\" is reserved for all hosts") }) + + t.Run("playbook with custom ssh shell set", func(t *testing.T) { + c, err := New("testdata/with-ssh-shell.yml", nil, nil) + require.NoError(t, err) + t.Logf("%+v", c) + assert.Equal(t, 1, len(c.Tasks), "single task") + assert.Equal(t, "umputun", c.User, "user") + assert.Equal(t, "/bin/bash", c.Tasks[0].Commands[0].SSHShell, "ssh shell") + + tsk := c.Tasks[0] + assert.Equal(t, 5, len(tsk.Commands), "5 commands") + assert.Equal(t, "deploy-remark42", tsk.Name, "task name") + }) + t.Run("playbook with custom ssh shell override", func(t *testing.T) { + c, err := New("testdata/with-ssh-shell.yml", &Overrides{SSHShell: "/bin/zsh"}, nil) + require.NoError(t, err) + t.Logf("%+v", c) + assert.Equal(t, 1, len(c.Tasks), "single task") + assert.Equal(t, "umputun", c.User, "user") + assert.Equal(t, "/bin/zsh", c.Tasks[0].Commands[0].SSHShell, "ssh shell") + + tsk := c.Tasks[0] + assert.Equal(t, 5, len(tsk.Commands), "5 commands") + assert.Equal(t, "deploy-remark42", tsk.Name, "task name") + }) } func TestPlayBook_Task(t *testing.T) { @@ -216,6 +243,18 @@ func TestPlayBook_Task(t *testing.T) { assert.Equal(t, 1, len(tsk.Commands)) assert.Equal(t, "ad-hoc", tsk.Name) assert.Equal(t, "echo 123", tsk.Commands[0].Script) + assert.Equal(t, "/bin/sh", tsk.Commands[0].SSHShell) + }) + + t.Run("adhoc with custom shell", func(t *testing.T) { + c, err := New("", &Overrides{AdHocCommand: "echo 123", User: "umputun", SSHShell: "/bin/zsh"}, nil) + require.NoError(t, err) + tsk, err := c.Task("ad-hoc") + require.NoError(t, err) + assert.Equal(t, 1, len(tsk.Commands)) + assert.Equal(t, "ad-hoc", tsk.Name) + assert.Equal(t, "echo 123", tsk.Commands[0].Script) + assert.Equal(t, "/bin/zsh", tsk.Commands[0].SSHShell) }) } @@ -480,6 +519,8 @@ hosts: // create test HTTP server ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch filepath.Ext(r.URL.Path) { + case ".bad": + w.WriteHeader(http.StatusInternalServerError) case ".toml": http.ServeFile(w, r, tomlFile.Name()) default: @@ -499,6 +540,7 @@ hosts: {"load YAML from URL without extension", ts.URL + "/inventory", false}, {"load TOML from file", tomlFile.Name(), false}, {"load TOML from URL", ts.URL + "/inventory.toml", false}, + {"load from URL with bad status", ts.URL + "/blah.bad", true}, {"invalid URL", "http://not-a-valid-url", true}, {"file not found", "nonexistent-file.yaml", true}, } diff --git a/pkg/config/testdata/with-ssh-shell.yml b/pkg/config/testdata/with-ssh-shell.yml new file mode 100644 index 00000000..5f3b8bd0 --- /dev/null +++ b/pkg/config/testdata/with-ssh-shell.yml @@ -0,0 +1,42 @@ +user: umputun +ssh_shell: /bin/bash + +targets: + remark42: + hosts: [{name: "h1", host: "h1.example.com"}, {host: "h2.example.com"}] + + +tasks: + - name: deploy-remark42 + commands: + - name: wait + script: sleep 5 + + - name: copy configuration + copy: {"src": "/local/remark42.yml", "dst": "/srv/remark42.yml", "mkdir": true} + + - name: some local command + options: {local: true} + script: | + ls -la /srv + du -hcs /srv + + - name: git + before: "echo before git" + after: "echo after git" + onerror: "echo onerror git" + script: | + git clone https://example.com/remark42.git /srv || true # clone if doesn't exists, but don't fail if exists + cd /srv + git pull + + - name: docker + options: {no_auto: true} + script: | + docker pull umputun/remark42:latest + docker stop remark42 || true + docker rm remark42 || true + docker run -d --name remark42 -p 8080:8080 umputun/remark42:latest + env: + FOO: bar + BAR: qux \ No newline at end of file diff --git a/pkg/executor/connector.go b/pkg/executor/connector.go index ec77fb4e..64c6b65b 100644 --- a/pkg/executor/connector.go +++ b/pkg/executor/connector.go @@ -118,3 +118,7 @@ func (c *Connector) sshConfig(user, privateKeyPath string) (*ssh.ClientConfig, e return sshConfig, nil } + +func (c *Connector) String() string { + return fmt.Sprintf("ssh connector with private key %s.., timeout %v, agent %v", c.privateKey[:8], c.timeout, c.enableAgent) +} diff --git a/pkg/executor/local.go b/pkg/executor/local.go index 66bafc3a..2ed93dc8 100644 --- a/pkg/executor/local.go +++ b/pkg/executor/local.go @@ -28,14 +28,24 @@ func (l *Local) SetSecrets(secrets []string) { // Run executes command on local hostAddr, inside the shell func (l *Local) Run(ctx context.Context, cmd string, opts *RunOpts) (out []string, err error) { + shell := func() string { + if strings.HasPrefix(cmd, "sh -c") { + return "sh" // command has sh -c prefix, so use sh + } + envShell := os.Getenv("SHELL") + if envShell == "" { + return "/bin/sh" // default to /bin/sh + } + return envShell // use SHELL env var + } - if strings.HasPrefix(cmd, "sh -c ") { + if strings.HasPrefix(cmd, shell()+" -c ") { // strip sh -c 'command' to just command to avoid double shell - cmd = strings.TrimPrefix(cmd, "sh -c ") + cmd = strings.TrimPrefix(cmd, shell()+" -c ") cmd = strings.TrimPrefix(cmd, "'") cmd = strings.TrimSuffix(cmd, "'") } - command := exec.CommandContext(ctx, "sh", "-c", cmd) + command := exec.CommandContext(ctx, shell(), "-c", cmd) //nolint outLog, errLog := MakeOutAndErrWriters("localhost", "", opts != nil && opts.Verbose, l.secrets) outLog.Write([]byte(cmd)) // nolint diff --git a/pkg/executor/remote_test.go b/pkg/executor/remote_test.go index 2763b44a..73378f4e 100644 --- a/pkg/executor/remote_test.go +++ b/pkg/executor/remote_test.go @@ -653,9 +653,10 @@ func startTestContainer(t *testing.T) (hostAndPort string, teardown func()) { require.NoError(t, err) req := testcontainers.ContainerRequest{ - Image: "lscr.io/linuxserver/openssh-server:latest", - ExposedPorts: []string{"2222/tcp"}, - WaitingFor: wait.NewLogStrategy("done.").WithStartupTimeout(time.Second * 60), + AlwaysPullImage: true, + Image: "lscr.io/linuxserver/openssh-server:latest", + ExposedPorts: []string{"2222/tcp"}, + WaitingFor: wait.NewLogStrategy("done.").WithStartupTimeout(time.Second * 60), Files: []testcontainers.ContainerFile{ {HostFilePath: "testdata/test_ssh_key.pub", ContainerFilePath: "/authorized_key"}, }, diff --git a/pkg/runner/commands.go b/pkg/runner/commands.go index 3ce2fd7b..d0955163 100644 --- a/pkg/runner/commands.go +++ b/pkg/runner/commands.go @@ -28,6 +28,7 @@ type execCmd struct { tsk *config.Task exec executor.Interface verbose bool + sshShell string } type execCmdResp struct { @@ -78,10 +79,10 @@ func (ec *execCmd) Script(ctx context.Context) (resp execCmdResp, err error) { resp.details = fmt.Sprintf(" {script: %s}", c) if ec.cmd.Options.Sudo { resp.details = fmt.Sprintf(" {script: %s, sudo: true}", c) - if strings.HasPrefix(c, "sh -c ") { // single line script already has sh -c + if strings.HasPrefix(c, ec.shell()+" ") { // single line script already has sh -c c = fmt.Sprintf("sudo %s", c) } else { - c = fmt.Sprintf("sudo sh -c %q", c) + c = fmt.Sprintf("sudo %s -c %q", ec.shell(), c) } } resp.verbose = scr @@ -289,11 +290,11 @@ func (ec *execCmd) Wait(ctx context.Context) (resp execCmdResp, err error) { resp.details = fmt.Sprintf(" {wait: %s, timeout: %v, duration: %v}", c, timeout.Truncate(100*time.Millisecond), duration.Truncate(100*time.Millisecond)) - waitCmd := fmt.Sprintf("sh -c %q", c) // run wait command in a shell + waitCmd := fmt.Sprintf("%s -c %q", ec.shell(), c) // run wait command in a shell if ec.cmd.Options.Sudo { resp.details = fmt.Sprintf(" {wait: %s, timeout: %v, duration: %v, sudo: true}", c, timeout.Truncate(100*time.Millisecond), duration.Truncate(100*time.Millisecond)) - waitCmd = fmt.Sprintf("sudo sh -c %q", c) // add sudo if needed + waitCmd = fmt.Sprintf("sudo %s -c %q", ec.shell(), c) // add sudo if needed } resp.verbose = script @@ -322,10 +323,10 @@ func (ec *execCmd) Echo(ctx context.Context) (resp execCmdResp, err error) { tmpl := templater{hostAddr: ec.hostAddr, hostName: ec.hostName, task: ec.tsk, command: ec.cmd.Name, env: ec.cmd.Environment} echoCmd := tmpl.apply(ec.cmd.Echo) if !strings.HasPrefix(echoCmd, "echo ") { - echoCmd = fmt.Sprintf("echo %s", echoCmd) + echoCmd = fmt.Sprintf("echo %q", echoCmd) } if ec.cmd.Options.Sudo { - echoCmd = fmt.Sprintf("sudo %s", echoCmd) + echoCmd = fmt.Sprintf("sudo %q", echoCmd) } out, err := ec.exec.Run(ctx, echoCmd, nil) if err != nil { @@ -355,7 +356,7 @@ func (ec *execCmd) checkCondition(ctx context.Context) (bool, error) { }() if ec.cmd.Options.Sudo { // command's sudo also applies to condition script - c = fmt.Sprintf("sudo sh -c %q", c) + c = fmt.Sprintf("sudo %s -c %q", ec.shell(), c) } // run the condition command @@ -428,7 +429,7 @@ func (ec *execCmd) prepScript(ctx context.Context, s string, r io.Reader) (cmd, if err = ec.exec.Upload(ctx, tmp.Name(), dst, &executor.UpDownOpts{Mkdir: true}); err != nil { return "", "", nil, ec.errorFmt("can't upload script to %s: %w", ec.hostAddr, err) } - cmd = fmt.Sprintf("sh -c %s", dst) + cmd = fmt.Sprintf("%s -c %s", ec.shell(), dst) teardown = func() error { // remove the temp dir with the script from the remote hostAddr, @@ -504,3 +505,16 @@ func (ec *execCmd) error(err error) *execCmdErr { func (ec *execCmd) errorFmt(format string, a ...any) *execCmdErr { return &execCmdErr{err: fmt.Errorf(format, a...), cmd: *ec} } + +func (ec *execCmd) shell() string { + if ec.sshShell == "" { + return "/bin/sh" + } + if ec.cmd.Options.Local { + if os.Getenv("SHELL") == "" { + return "/bin/sh" // default to /bin/sh if SHELL env var is not set + } + return os.Getenv("SHELL") // local commands always use local sh + } + return ec.sshShell +} diff --git a/pkg/runner/commands_test.go b/pkg/runner/commands_test.go index 46673294..3fe7537e 100644 --- a/pkg/runner/commands_test.go +++ b/pkg/runner/commands_test.go @@ -292,7 +292,7 @@ func Test_execCmd(t *testing.T) { Script: "echo condition true", Name: "test"}} resp, err := ec.Script(ctx) require.NoError(t, err) - assert.Equal(t, " {script: sh -c 'echo condition true'}", resp.details) + assert.Equal(t, " {script: /bin/sh -c 'echo condition true'}", resp.details) }) t.Run("condition true inverted", func(t *testing.T) { @@ -426,7 +426,7 @@ func Test_execCmdWithTmp(t *testing.T) { resp, err := ec.Script(ctx) require.NoError(t, err) // {script: sh -c /tmp/.spot-8420993611669644288/spot-script1149755050, sudo: true} - assert.Contains(t, resp.details, " {script: sh -c /tmp/.spot-") + assert.Contains(t, resp.details, " {script: /bin/sh -c /tmp/.spot-") assert.Contains(t, resp.details, ", sudo: true}") // [INFO] deleted recursively /tmp/.spot-8279767396215533568 diff --git a/pkg/runner/runner.go b/pkg/runner/runner.go index d12ccb58..a9e83750 100644 --- a/pkg/runner/runner.go +++ b/pkg/runner/runner.go @@ -33,6 +33,7 @@ type Process struct { ColorWriter *executor.ColorizedWriter Verbose bool Dry bool + SSHShell string Skip []string Only []string @@ -191,7 +192,8 @@ func (p *Process) runTaskOnHost(ctx context.Context, tsk *config.Task, hostAddr, log.Printf("[INFO] %s", p.infoMessage(cmd, hostAddr, hostName)) stCmd := time.Now() - ec := execCmd{cmd: cmd, hostAddr: hostAddr, hostName: hostName, tsk: &activeTask, exec: remote, verbose: p.Verbose} + ec := execCmd{cmd: cmd, hostAddr: hostAddr, hostName: hostName, tsk: &activeTask, exec: remote, + verbose: p.Verbose, sshShell: p.SSHShell} ec = p.pickCmdExecutor(cmd, ec, hostAddr, hostName) // pick executor on dry run or local command exResp, err := p.execCommand(ctx, ec) @@ -313,9 +315,10 @@ func (p *Process) onError(ctx context.Context, err error) { ec := execCmd{ tsk: execErr.cmd.tsk, cmd: config.Cmd{ - Name: execErr.cmd.tsk.Name, - Script: execErr.cmd.tsk.OnError, - Options: config.CmdOptions{Local: true}, // force local execution for on-error command + Name: execErr.cmd.tsk.Name, + Script: execErr.cmd.tsk.OnError, + Options: config.CmdOptions{Local: true}, // force local execution for on-error command + SSHShell: "/bin/sh", // local run always with /bin/sh }, hostAddr: execErr.cmd.hostAddr, hostName: execErr.cmd.hostName, diff --git a/pkg/runner/runner_test.go b/pkg/runner/runner_test.go index 794b56aa..52d4d2d0 100644 --- a/pkg/runner/runner_test.go +++ b/pkg/runner/runner_test.go @@ -347,6 +347,7 @@ func TestProcess_RunWithSudo(t *testing.T) { Playbook: conf, ColorWriter: executor.NewColorizedWriter(os.Stdout, "", "", "", nil), Only: []string{"root only single line with var"}, + SSHShell: "/bin/sh", } outWriter := &bytes.Buffer{} @@ -355,7 +356,7 @@ func TestProcess_RunWithSudo(t *testing.T) { require.NoError(t, err) assert.Equal(t, 1, res.Commands) assert.Equal(t, 1, res.Hosts) - assert.Contains(t, outWriter.String(), " > sudo sh -c 'vvv=123 && echo var=$vvv'") + assert.Contains(t, outWriter.String(), " > sudo /bin/sh -c 'vvv=123 && echo var=$vvv'") assert.Contains(t, outWriter.String(), " > var=123") }) diff --git a/spot.1 b/spot.1 index 92533db7..365a0710 100644 --- a/spot.1 +++ b/spot.1 @@ -1,5 +1,5 @@ -.TH "SPOT" 1 v1.9.2 20230718T142230 spot manual -.\" Automatically generated by Pandoc 3.1.2 +.TH "SPOT" 1 20230723T201928 spot manual +.\" Automatically generated by Pandoc 3.1.6 .\" .\" Define V font for inline verbatim, using C font in formats .\" that render this, and otherwise B font. @@ -167,9 +167,20 @@ Defaults to \f[V]1\f[R], which means hosts will be handled sequentially. .IP \[bu] 2 \f[V]--timeout\f[R]: Sets the SSH timeout. Defaults to \f[V]30s\f[R]. -You can also set the environment variable \f[V]$SPOT_TIMEOUT\f[R] to +User can also set the environment variable \f[V]$SPOT_TIMEOUT\f[R] to define the SSH timeout. .IP \[bu] 2 +\f[V]--ssh-agent\f[R]: Enables the use of the SSH agent for +authentication. +Defaults to \f[V]false\f[R]. +User can also set the environment variable \f[V]SPOT_SSH_AGENT\f[R] to +define the value. +.IP \[bu] 2 +\f[V]--shell\f[R] - shell to use for remote ssh execution, default is +\f[V]/bin/sh\f[R]. +User can also set the environment variable \f[V]SPOT_SHELL\f[R] to +define the value. +.IP \[bu] 2 \f[V]-i\f[R], \f[V]--inventory=\f[R]: Specifies the inventory file or url to use for the task execution. Overrides the inventory file defined in the playbook file. @@ -200,6 +211,11 @@ Providing the \f[V]-e\f[R] flag multiple times with different environment variables sets multiple environment variables, e.g., \f[V]-e VAR1:VALUE1 -e VAR2:VALUE2\f[R]. .IP \[bu] 2 +\f[V]-E\f[R], \f[V]--env-file=\f[R]: Sets the environment variables from +file to be used during the task execution. +Default is env.yml. +Can be also set with environment variable \f[V]SPOT_ENV_FILE\f[R]. +.IP \[bu] 2 \f[V]--dry\f[R]: Enables dry-run mode, which prints out the commands to be executed without actually executing them. .IP \[bu] 2 @@ -245,6 +261,7 @@ hosts and groups of hosts on which a task can be executed. \f[C] user: umputun # default ssh user. Can be overridden by -u flag or by inventory or host definition ssh_key: keys/id_rsa # ssh key +ssh_shell: /bin/bash # shell to use for remote ssh execution, default is /bin/sh inventory: /etc/spot/inventory.yml # default inventory file. Can be overridden by --inventory flag # list of targets, i.e. hosts, inventory files or inventory URLs @@ -318,6 +335,7 @@ is easier to read and write, but also more limited in its capabilities. \f[C] user: umputun # default ssh user. Can be overridden by -u flag or by inventory or host definition ssh_key: keys/id_rsa # ssh key +ssh_shell: /bin/bash # shell to use for remote ssh execution, default is /bin/sh inventory: /etc/spot/inventory.yml # default inventory file. Can be overridden by --inventory flag targets: [\[dq]devbox1\[dq], \[dq]devbox2\[dq], \[dq]h1.example.com:2222\[dq], \[dq]h2.example.com\[dq]] # list of host names from inventory and direct host ips @@ -477,6 +495,11 @@ script: | echo all good, 123 \f[R] .fi +.PP +Read more about YAML multiline string formatting on +yaml-multiline.info (https://yaml-multiline.info/) and this +stackoverflow +post (https://stackoverflow.com/questions/3790454/how-do-i-break-a-string-in-yaml-over-multiple-lines). .SS \f[V]copy\f[R] .PP Copies a file from the local machine to the remote host(s). @@ -760,6 +783,40 @@ commands: copy: {src: $FILE_NAME, dest: /tmp/file2} \f[R] .fi +.SS Setting environment variables +.PP +Environment variables can be set with \f[V]--env\f[R] / \f[V]-e\f[R] cli +option. +For example: \f[V]-e VAR1:VALUE1 -e VAR2:VALUE2\f[R]. +Environment variables can also be set in the environment file (default +\f[V]env.yml\f[R] can be changed with \f[V]--env-file\f[R] / +\f[V]-E\f[R] cli flag). +For example: +.IP +.nf +\f[C] +vars: + VAR1: VALUE1 + VAR2: VALUE2 +\f[R] +.fi +.PP +Environment variable can be used in the playbook file with the expected +syntax: \f[V]$VAR_NAME\f[R] or \f[V]${VAR_NAME}\f[R]. +For example: +.IP +.nf +\f[C] +commands: + - name: some command + script: echo $VAR1 + - name: another command + copy: {\[dq]src\[dq]: \[dq]testdata/*.csv\[dq], \[dq]dst\[dq]: \[dq]$VAR2\[dq]} +\f[R] +.fi +.PP +In case of a conflict between environment variables set in the +environment file and the cli, the cli variables will take precedence. .SS Targets .PP Targets are used to define the remote hosts to execute the tasks on. @@ -1255,6 +1312,21 @@ default AWS credential. This means that the provider will use the credentials from the environment variables \f[V]AWS_ACCESS_KEY_ID\f[R] and \f[V]AWS_SECRET_ACCESS_KEY\f[R]. +.SS Ansible Vault Secrets Provider +.PP +Spot gives ability to use full encrypted \f[V]YAML\f[R] files by +ansbile-vault (https://docs.ansible.com/ansible/latest/cli/ansible-vault.html) +.PP +\f[V]--secrets.provider=ansible-vault\f[R]: selects the Ansible Vault +secrets provider. +\f[V]--secrets.ansible.path\f[R] or \f[V]$SPOT_SECRETS_ANSIBLE_PATH\f[R] +path to the ansible-vault file \f[V]--secrets.ansible.secret\f[R] or +\f[V]$SPOT_SECRETS_ANSIBLE_SECRET\f[R] secret string for decrypting +ansible-vault file +.PP +note: encrypted values in the vault should be in next format +\f[V]key[string]:value[string]\f[R] without nested \f[V]lists\f[R] and +\f[V]maps\f[R]. .SS Managing Secrets with \f[V]spot-secrets\f[R] .PP Spot provides a simple way to manage secrets for builtin provider using