Skip to content

Commit

Permalink
add basic support of toml for both playbook and inventory
Browse files Browse the repository at this point in the history
  • Loading branch information
@umputun
umputun committed May 2, 2023
1 parent bee255f commit a826221
Show file tree
Hide file tree
Showing 40 changed files with 7,492 additions and 82 deletions.
7 changes: 6 additions & 1 deletion README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ Spot (aka `simplotask`) is a powerful and easy-to-use tool for effortless deploy

- Define tasks with a list of commands and the list of target hosts.
- Support for remote hosts specified directly or through inventory files/URLs.
- Everything can be defined in a simple YAML file.
- Everything can be defined in a simple YAML or TOML file.
- Run scripts on remote hosts as well as on the localhost.
- Built-in commands: copy, sync, delete and wait.
- Concurrent execution of task on multiple hosts.
Expand Down Expand Up @@ -120,6 +120,9 @@ tasks:
- wait: {cmd: "curl -s localhost:8080/health", timeout: "10s", interval: "1s"} # wait for health check to pass
```

*Alternatively, the playbook can be represented using the TOML format.*


## Task details

Each task consists of a list of commands that will be executed on the remote host(s). The task can also define the following optional fields:
Expand Down Expand Up @@ -298,6 +301,8 @@ This format is useful when you want to define a list of hosts without groups.

In each case inventory automatically merged and a special group `all` will be created that contains all the hosts.

*Alternatively, the inventory can be represented using the TOML format.*

## Runtime variables

Spot supports runtime variables that can be used in the playbook file. The following variables are supported:
Expand Down
111 changes: 64 additions & 47 deletions app/config/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,90 +12,91 @@ import (
"strings"
"time"

"github.com/pelletier/go-toml/v2"
"gopkg.in/yaml.v3"

"github.com/umputun/simplotask/app/config/deepcopy"
)

// PlayBook defines top-level config yaml
type PlayBook struct {
User string `yaml:"user"` // ssh user
SSHKey string `yaml:"ssh_key"` // ssh key
Inventory string `yaml:"inventory"` // inventory file or url
Targets map[string]Target `yaml:"targets"` // list of targets/environments
Tasks []Task `yaml:"tasks"` // list of tasks
User string `yaml:"user" toml:"user"` // ssh user
SSHKey string `yaml:"ssh_key" toml:"ssh_key"` // ssh key
Inventory string `yaml:"inventory" toml:"inventory"` // inventory file or url
Targets map[string]Target `yaml:"targets" toml:"targets"` // list of targets/environments
Tasks []Task `yaml:"tasks" toml:"tasks"` // list of tasks

inventory *InventoryData // loaded inventory
overrides *Overrides // overrides passed from cli
}

// Target defines hosts to run commands on
type Target struct {
Name string `yaml:"name"`
Hosts []Destination `yaml:"hosts"` // direct list of hosts to run commands on, no need to use inventory
Groups []string `yaml:"groups"` // list of groups to run commands on, matches to inventory
Names []string `yaml:"names"` // list of host names to run commands on, matches to inventory
Name string `yaml:"name" toml:"name"` // name of target
Hosts []Destination `yaml:"hosts" toml:"hosts"` // direct list of hosts to run commands on, no need to use inventory
Groups []string `yaml:"groups" toml:"groups"` // list of groups to run commands on, matches to inventory
Names []string `yaml:"names" toml:"names"` // list of host names to run commands on, matches to inventory
}

// Task defines multiple commands runs together
type Task struct {
Name string // name of target, set by config caller
User string `yaml:"user"`
SSHKey string `yaml:"ssh_key"`
Commands []Cmd `yaml:"commands"`
OnError string `yaml:"on_error"`
Name string `yaml:"name" toml:"name"` // name of target, set by config caller
User string `yaml:"user" toml:"user"`
SSHKey string `yaml:"ssh_key" toml:"ssh_key"`
Commands []Cmd `yaml:"commands" toml:"commands"`
OnError string `yaml:"on_error" toml:"on_error"`
}

// Cmd defines a single command
type Cmd struct {
Name string `yaml:"name"`
Copy CopyInternal `yaml:"copy"`
Sync SyncInternal `yaml:"sync"`
Delete DeleteInternal `yaml:"delete"`
Wait WaitInternal `yaml:"wait"`
Script string `yaml:"script"`
Environment map[string]string `yaml:"env"`
Name string `yaml:"name" toml:"name"`
Copy CopyInternal `yaml:"copy" toml:"copy"`
Sync SyncInternal `yaml:"sync" toml:"sync"`
Delete DeleteInternal `yaml:"delete" toml:"delete"`
Wait WaitInternal `yaml:"wait" toml:"wait"`
Script string `yaml:"script" toml:"script,multiline"`
Environment map[string]string `yaml:"env" toml:"env"`
Options struct {
IgnoreErrors bool `yaml:"ignore_errors"`
NoAuto bool `yaml:"no_auto"`
Local bool `yaml:"local"`
} `yaml:"options"`
IgnoreErrors bool `yaml:"ignore_errors" toml:"ignore_errors"`
NoAuto bool `yaml:"no_auto" toml:"no_auto"`
Local bool `yaml:"local" toml:"local"`
} `yaml:"options" toml:"options,omitempty"`
}

// Destination defines destination info
type Destination struct {
Name string `yaml:"name"`
Host string `yaml:"host"`
Port int `yaml:"port"`
User string `yaml:"user"`
Tags []string `yaml:"tags"`
Name string `yaml:"name" toml:"name"`
Host string `yaml:"host" toml:"host"`
Port int `yaml:"port" toml:"port"`
User string `yaml:"user" toml:"user"`
Tags []string `yaml:"tags" toml:"tags"`
}

// CopyInternal defines copy command, implemented internally
type CopyInternal struct {
Source string `yaml:"src"`
Dest string `yaml:"dst"`
Mkdir bool `yaml:"mkdir"`
Source string `yaml:"src" toml:"src"`
Dest string `yaml:"dst" toml:"dst"`
Mkdir bool `yaml:"mkdir" toml:"mkdir"`
}

// SyncInternal defines sync command (recursive copy), implemented internally
type SyncInternal struct {
Source string `yaml:"src"`
Dest string `yaml:"dst"`
Delete bool `yaml:"delete"`
Source string `yaml:"src" toml:"src"`
Dest string `yaml:"dst" toml:"dst"`
Delete bool `yaml:"delete" toml:"delete"`
}

// DeleteInternal defines delete command, implemented internally
type DeleteInternal struct {
Location string `yaml:"path"`
Recursive bool `yaml:"recur"`
Location string `yaml:"path" toml:"path"`
Recursive bool `yaml:"recur" toml:"recur"`
}

// WaitInternal defines wait command, implemented internally
type WaitInternal struct {
Timeout time.Duration `yaml:"timeout"`
CheckDuration time.Duration `yaml:"interval"`
Command string `yaml:"cmd"`
Timeout time.Duration `yaml:"timeout" toml:"timeout"`
CheckDuration time.Duration `yaml:"interval" toml:"interval"`
Command string `yaml:"cmd" toml:"cmd,multiline"`
}

// Overrides defines override for task passed from cli
Expand All @@ -108,8 +109,8 @@ type Overrides struct {

// InventoryData defines inventory data format
type InventoryData struct {
Groups map[string][]Destination `yaml:"groups"`
Hosts []Destination `yaml:"hosts"`
Groups map[string][]Destination `yaml:"groups" toml:"groups"`
Hosts []Destination `yaml:"hosts" toml:"hosts"`
}

const allHostsGrp = "all"
Expand Down Expand Up @@ -141,8 +142,17 @@ func New(fname string, overrides *Overrides) (res *PlayBook, err error) {
return nil, fmt.Errorf("can't read config %s: %w", fname, err)
}

if err = yaml.Unmarshal(data, res); err != nil {
return nil, fmt.Errorf("can't unmarshal config %s: %w", fname, err)
switch {
case strings.HasSuffix(fname, ".yml") || strings.HasSuffix(fname, ".yaml") || !strings.Contains(fname, "."):
if err = yaml.Unmarshal(data, res); err != nil {
return nil, fmt.Errorf("can't unmarshal config %s: %w", fname, err)
}
case strings.HasSuffix(fname, ".toml"):
if err = toml.Unmarshal(data, res); err != nil {
return nil, fmt.Errorf("can't unmarshal config %s: %w", fname, err)
}
default:
return nil, fmt.Errorf("unknown config format %s", fname)
}

if err = res.checkConfig(); err != nil {
Expand Down Expand Up @@ -408,8 +418,15 @@ func (p *PlayBook) loadInventory(loc string) (*InventoryData, error) {
defer rdr.Close() // nolint

var data InventoryData
if err := yaml.NewDecoder(rdr).Decode(&data); err != nil {
return nil, fmt.Errorf("inventory decoder failed: %w", err)
if !strings.HasSuffix(loc, ".toml") {
// we assume it is yaml. Can't do strict check, as we can have urls unrelated to file extension
if err = yaml.NewDecoder(rdr).Decode(&data); err != nil {
return nil, fmt.Errorf("can't parse inventory %s: %w", loc, err)
}
} else {
if err = toml.NewDecoder(rdr).Decode(&data); err != nil {
return nil, fmt.Errorf("can't parse inventory %s: %w", loc, err)
}
}

if len(data.Groups) > 0 {
Expand Down
88 changes: 54 additions & 34 deletions app/config/config_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"testing"

Expand All @@ -26,6 +27,18 @@ func TestNew(t *testing.T) {
assert.Equal(t, "deploy-remark42", tsk.Name, "task name")
})

t.Run("good toml file", func(t *testing.T) {
c, err := New("testdata/f1.toml", nil)
require.NoError(t, err)
t.Logf("%+v", c)
assert.Equal(t, 1, len(c.Tasks), "single task")
assert.Equal(t, "umputun", c.User, "user")

tsk := c.Tasks[0]
assert.Equal(t, 5, len(tsk.Commands), "5 commands")
assert.Equal(t, "deploy-remark42", tsk.Name, "task name")
})

t.Run("inventory from env", func(t *testing.T) {
err := os.Setenv("SPOT_INVENTORY", "testdata/hosts-with-groups.yml")
require.NoError(t, err)
Expand Down Expand Up @@ -444,12 +457,8 @@ func TestTargetHosts(t *testing.T) {
}

func TestPlayBook_loadInventory(t *testing.T) {
// create temporary inventory file
tmpFile, err := os.CreateTemp("", "inventory-*.yaml")
require.NoError(t, err)
defer os.Remove(tmpFile.Name())

_, err = tmpFile.WriteString(`---
// create temporary inventory files
yamlData := []byte(`
groups:
group1:
- host: example.com
Expand All @@ -459,43 +468,58 @@ groups:
hosts:
- {host: one.example.com, port: 2222}
`)
require.NoError(t, err)
yamlFile, _ := os.CreateTemp("", "inventory-*.yaml")
defer os.Remove(yamlFile.Name())
_ = os.WriteFile(yamlFile.Name(), yamlData, 0o644)

tomlData := []byte(`
[groups]
group1 = [
{ host = "example.com", port = 22 },
]
group2 = [
{ host = "another.com" },
]
[[hosts]]
host = "one.example.com"
port = 2222
`)
tomlFile, _ := os.CreateTemp("", "inventory-*.toml")
defer os.Remove(tomlFile.Name())
_ = os.WriteFile(tomlFile.Name(), tomlData, 0o644)

// create test HTTP server
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
http.ServeFile(w, r, tmpFile.Name())
switch filepath.Ext(r.URL.Path) {
case ".toml":
http.ServeFile(w, r, tomlFile.Name())
default:
http.ServeFile(w, r, yamlFile.Name())
}
}))
defer ts.Close()

// create test cases
testCases := []struct {
name string
loc string
expectError bool
}{
{
name: "load from file",
loc: tmpFile.Name(),
},
{
name: "load from url",
loc: ts.URL,
},
{
name: "invalid url",
loc: "http://not-a-valid-url",
expectError: true,
},
{
name: "file not found",
loc: "nonexistent-file.yaml",
expectError: true,
},
{"load YAML from file", yamlFile.Name(), false},
{"load YAML from URL", ts.URL + "/inventory.yaml", false},
{"load YAML from URL without extension", ts.URL + "/inventory", false},
{"load TOML from file", tomlFile.Name(), false},
{"load TOML from URL", ts.URL + "/inventory.toml", false},
{"invalid URL", "http://not-a-valid-url", true},
{"file not found", "nonexistent-file.yaml", true},
}

// run test cases
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
p := &PlayBook{
User: "testuser",
}
p := &PlayBook{User: "testuser"}
inv, err := p.loadInventory(tc.loc)

if tc.expectError {
Expand All @@ -508,29 +532,25 @@ hosts:
require.Len(t, inv.Groups, 3)
require.Len(t, inv.Hosts, 1)

// check "all" group
allGroup := inv.Groups["all"]
require.Len(t, allGroup, 3, "all group should contain all hosts")
require.Len(t, allGroup, 3)
assert.Equal(t, "another.com", allGroup[0].Host)
assert.Equal(t, 22, allGroup[0].Port)
assert.Equal(t, "example.com", allGroup[1].Host)
assert.Equal(t, 22, allGroup[1].Port)
assert.Equal(t, "one.example.com", allGroup[2].Host)
assert.Equal(t, 2222, allGroup[2].Port)

// check "group1"
group1 := inv.Groups["group1"]
require.Len(t, group1, 1)
assert.Equal(t, "example.com", group1[0].Host)
assert.Equal(t, 22, group1[0].Port)

// check "group2"
group2 := inv.Groups["group2"]
require.Len(t, group2, 1)
assert.Equal(t, "another.com", group2[0].Host)
assert.Equal(t, 22, group2[0].Port)

// check hosts
assert.Equal(t, "one.example.com", inv.Hosts[0].Host)
assert.Equal(t, 2222, inv.Hosts[0].Port)
})
Expand Down
Loading

0 comments on commit a826221

Please sign in to comment.