Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Targets refactor #80

Merged
merged 5 commits into from
May 10, 2023
Merged

Targets refactor #80

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
d8b3723
introduce target service to get destinations
umputun May 10, 2023
c1a7783
refactor target func to small logical parts, add test
umputun May 10, 2023
39b1134
lint: address dedup res shadowing
umputun May 10, 2023
b3e7931
rename target struct for better clarity, add descriptive comments for…
umputun May 10, 2023
293b8d4
add target tests for user and custom port
umputun May 10, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion pkg/config/command.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -191,7 +191,7 @@ func (cmd *Cmd) genSecrets() []string {

// UnmarshalYAML implements yaml.Unmarshaler interface
// It allows to unmarshal a "copy" from a single field or a slice
// All other fields are unmarshalled as usual. Limited to string, in, struct, slice or map
// All other fields are unmarshalled as usual. Limited to string, int, struct, slice or map
func (cmd *Cmd) UnmarshalYAML(unmarshal func(interface{}) error) error {
var asMap map[string]interface{}
if err := unmarshal(&asMap); err != nil {
Expand Down Expand Up @@ -268,6 +268,8 @@ func (cmd *Cmd) UnmarshalYAML(unmarshal func(interface{}) error) error {
return nil
}

// validate checks if a Cmd has exactly one command type set (script, copy, mcopy, delete, sync, or wait)
// and returns an error if there are either multiple command types set or none set.
func (cmd *Cmd) validate() error {
cmdTypes := []struct {
name string
Expand Down
212 changes: 41 additions & 171 deletions pkg/config/playbook.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,14 @@ type SimplePlayBook struct {
Task []Cmd `yaml:"task" toml:"task"` // single task is a list of commands
}

// Task defines multiple commands runs together
type Task struct {
Name string `yaml:"name" toml:"name"` // name of target, set by config caller
User string `yaml:"user" toml:"user"`
Commands []Cmd `yaml:"commands" toml:"commands"`
OnError string `yaml:"on_error" toml:"on_error"`
}

// Target defines hosts to run commands on
type Target struct {
Name string `yaml:"name" toml:"name"` // name of target
Expand All @@ -59,14 +67,6 @@ type Target struct {
Tags []string `yaml:"tags" toml:"tags"` // list of tags to run commands on, matches to inventory
}

// Task defines multiple commands runs together
type Task struct {
Name string `yaml:"name" toml:"name"` // name of target, set by config caller
User string `yaml:"user" toml:"user"`
Commands []Cmd `yaml:"commands" toml:"commands"`
OnError string `yaml:"on_error" toml:"on_error"`
}

// Destination defines destination info
type Destination struct {
Name string `yaml:"name" toml:"name"`
Expand Down Expand Up @@ -95,7 +95,11 @@ const (
inventoryEnv = "SPOT_INVENTORY"
)

// New makes new config from yml
// New creates a new PlayBook instance by loading the playbook configuration from the specified file. If the file cannot be
// found, and an ad-hoc command is specified in the overrides, a fake playbook with the ad-hoc command is created.
// The method also loads any secrets from the specified secrets provider and the inventory data from the specified
// location (if set). Returns an error if the playbook configuration cannot be loaded or parsed,
// or if the inventory data cannot be loaded.
func New(fname string, overrides *Overrides, secProvider SecretsProvider) (res *PlayBook, err error) {
log.Printf("[DEBUG] request to load playbook %q", fname)
res = &PlayBook{
Expand Down Expand Up @@ -243,7 +247,11 @@ func unmarshalPlaybookFile(fname string, data []byte, overrides *Overrides, res
return multierror.Append(errors, err).Unwrap()
}

// Task returns task by name
// Task returns the task with the specified name from the playbook's list of tasks. If the name is "ad-hoc" and an ad-hoc
// command is specified in the playbook's overrides, a fake task with a single command is created.
// The method performs a deep copy of the task to avoid side effects of overrides on the original config and also applies
// any overrides for the user and environment variables to the task and its commands.
// Returns an error if the task cannot be found or copied.
func (p *PlayBook) Task(name string) (*Task, error) {
searchTask := func(tsk []Task, name string) (*Task, error) {
if name == "ad-hoc" && p.overrides.AdHocCommand != "" {
Expand Down Expand Up @@ -297,35 +305,11 @@ func (p *PlayBook) Task(name string) (*Task, error) {
return res, nil
}

// AllSecretValues returns all secret values from all tasks and all commands.
// It is used to mask Secrets in logs.
func (p *PlayBook) AllSecretValues() []string {
res := make([]string, 0, len(p.secrets))
for _, v := range p.secrets {
res = append(res, v)
}
sort.Strings(res)
return res
}

// TargetHosts returns target hosts for given target name.
// After it gets destinations from targetHosts(name) it applies overrides of user, set default port 22 if needed
// and deduplicate results.
func (p *PlayBook) TargetHosts(name string) ([]Destination, error) {

dedup := func(in []Destination) []Destination {
var res []Destination
seen := make(map[string]struct{})
for _, d := range in {
key := d.Host + ":" + strconv.Itoa(d.Port) + ":" + d.User
if _, ok := seen[key]; !ok {
seen[key] = struct{}{}
res = append(res, d)
}
}
return res
}

userOverride := func(u string) string {
// apply overrides of user
if p.overrides != nil && p.overrides.User != "" {
Expand All @@ -339,7 +323,8 @@ func (p *PlayBook) TargetHosts(name string) ([]Destination, error) {
return p.User
}

res, err := p.targetHosts(name)
tgExtractor := newTargetExtractor(p.Targets, p.User, p.inventory)
res, err := tgExtractor.Destinations(name)
if err != nil {
return nil, err
}
Expand All @@ -352,146 +337,27 @@ func (p *PlayBook) TargetHosts(name string) ([]Destination, error) {
res[i] = h
}

return dedup(res), nil
return res, nil
}

// targetHosts returns target hosts for given target name.
// The result is not deduplicated and not modified with overrides.
func (p *PlayBook) targetHosts(name string) ([]Destination, error) {
t, ok := p.Targets[name] // get target from playbook
if ok {
if len(t.Hosts) == 0 && len(t.Names) == 0 && len(t.Groups) == 0 && len(t.Tags) == 0 {
return nil, fmt.Errorf("target %q has no hosts, names, tags or groups", name)
}
log.Printf("[DEBUG] target %q found in playbook", name)
// we have found target in playbook, process hosts, names and group
res := []Destination{}

if len(t.Hosts) > 0 {
// target has "hosts", use all of them as is
res = append(res, t.Hosts...)
log.Printf("[DEBUG] target %q has %d hosts: %+v", name, len(t.Hosts), t.Hosts)
}

if len(t.Names) > 0 && p.inventory != nil {
// target has "names", match them to "all" group in inventory by name
for _, n := range t.Names {
for _, h := range p.inventory.Groups[allHostsGrp] {
if strings.EqualFold(h.Name, n) {
res = append(res, h)
log.Printf("[DEBUG] target %q found name match %+v", name, h)
break
}
}
}
}

if len(t.Groups) > 0 && p.inventory != nil {
// target has "groups", get all hosts from inventory for each group
for _, g := range t.Groups {
// we don't set default port and user here, as they are set in inventory already
res = append(res, p.inventory.Groups[g]...)
log.Printf("[DEBUG] target %q found group match %+v", name, p.inventory.Groups[g])
}
}

if len(t.Tags) > 0 && p.inventory != nil {
// target has "tags", get all hosts from inventory for each tag
for _, tag := range t.Tags {
for _, h := range p.inventory.Groups[allHostsGrp] {
if len(h.Tags) == 0 {
continue
}
for _, t := range h.Tags {
if strings.EqualFold(t, tag) {
res = append(res, h)
log.Printf("[DEBUG] target %q found tag match %+v", name, h)
}
}
}
}
}

if len(res) == 0 {
return nil, fmt.Errorf("hosts for target %q not found", name)
}
log.Printf("[DEBUG] target %q has %d total hosts: %+v", name, len(res), res)
return res, nil
}

// target not defined in playbook
log.Printf("[DEBUG] target %q not found in playbook", name)

// try first as group in inventory
hosts, ok := p.inventory.Groups[name]
if ok {
res := make([]Destination, len(hosts))
copy(res, hosts)
log.Printf("[DEBUG] target %q found as group in inventory: %+v", name, res)
return res, nil
}

// try as a tag in inventory
res := []Destination{}
for _, h := range p.inventory.Groups[allHostsGrp] {
if len(h.Tags) == 0 {
continue
}
for _, t := range h.Tags {
if strings.EqualFold(t, name) {
res = append(res, h)
}
}
}
if len(res) > 0 {
log.Printf("[DEBUG] target %q found as tag in inventory: %+v", name, res)
return res, nil
}

// try as single host name in inventory
for _, h := range p.inventory.Groups[allHostsGrp] {
if strings.EqualFold(h.Name, name) {
log.Printf("[DEBUG] target %q found as name in inventory: %+v", name, h)
return []Destination{h}, nil
}
}

// try as a single host address in inventory
for _, h := range p.inventory.Groups[allHostsGrp] {
if strings.EqualFold(h.Host, name) {
log.Printf("[DEBUG] target %q found as host in inventory: %+v", name, h)
return []Destination{h}, nil
}
}

// try as single host or host:port or user@host:port
user := p.User
if strings.Contains(name, "@") { // extract user from name
elems := strings.Split(name, "@")
user = elems[0]
if len(elems) > 1 {
name = elems[1] // skip user part
}
}

if strings.Contains(name, ":") {
elems := strings.Split(name, ":")
port, err := strconv.Atoi(elems[1])
if err != nil {
return nil, fmt.Errorf("can't parse port %s: %w", elems[1], err)
}
log.Printf("[DEBUG] target %q used as host:port %s:%d", name, elems[0], port)
return []Destination{{Host: elems[0], Port: port, User: user}}, nil
// AllSecretValues returns all secret values from all tasks and all commands.
// It is used to mask Secrets in logs.
func (p *PlayBook) AllSecretValues() []string {
res := make([]string, 0, len(p.secrets))
for _, v := range p.secrets {
res = append(res, v)
}

// finally we assume it is a host name, with default port 22
log.Printf("[DEBUG] target %q used as host:22 %s", name, name)
return []Destination{{Host: name, Port: 22, User: user}}, nil
sort.Strings(res)
return res
}

// loadInventoryFile loads inventory from file or url and returns a struct with groups.
// Hosts, if presented, are loaded to the group "all". All the other groups are loaded to "all"
// as well and also to their own group.
// loadInventory loads the inventory data from the specified location (file or URL) and returns it as an InventoryData struct.
// The inventory data is parsed as either YAML or TOML, depending on the file extension.
// The method also performs some additional processing on the inventory data:
// - It creates a group "all" that contains all hosts from all groups.
// - It sorts the hosts in the "all" group by host name for predictable order in tests and processing.
// - It sets default port and user values for all inventory groups if not already set.
// Returns an error if the inventory data cannot be loaded or parsed, or if the "all" group is reserved for all hosts.
func (p *PlayBook) loadInventory(loc string) (*InventoryData, error) {

reader := func(loc string) (r io.ReadCloser, err error) {
Expand Down Expand Up @@ -578,7 +444,11 @@ func (p *PlayBook) loadInventory(loc string) (*InventoryData, error) {
return &data, nil
}

// checkConfig checks validity of config
// checkConfig validates the PlayBook configuration by ensuring that:
// - all tasks have unique names and no empty names
// - all commands have a single type set
// - the target set is not called "all"
// Returns an error if any of these conditions are not met.
func (p *PlayBook) checkConfig() error {

// check that all tasks have unique names in the playbook and no empty names
Expand Down
Loading